From a95184b5b6afde06df885843a95ad8adfa1e677f Mon Sep 17 00:00:00 2001 From: wangbo Date: Fri, 31 Jul 2026 22:49:05 +0800 Subject: [PATCH] =?UTF-8?q?fix(acceptance):=20=E9=98=BB=E6=96=AD=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E6=8E=A7=E5=88=B6=E9=9D=A2=E6=BC=82=E7=A7=BB=E6=B1=A1?= =?UTF-8?q?=E6=9F=93=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。 --- apps/api/cmd/acceptance-bootstrap/main.go | 82 ++++- apps/api/cmd/acceptance-load/main.go | 77 +++-- apps/api/cmd/acceptance-load/main_test.go | 30 ++ apps/api/docs/swagger.json | 54 +++ apps/api/docs/swagger.yaml | 36 ++ apps/api/internal/auth/auth.go | 23 +- .../auth/auth_store_unavailable_test.go | 31 ++ apps/api/internal/httpapi/compat_protocol.go | 4 + .../internal/httpapi/compat_protocol_test.go | 34 ++ apps/api/internal/httpapi/gemini_compat.go | 1 + apps/api/internal/httpapi/handlers.go | 1 + .../httpapi/volces_compat_handlers.go | 2 + .../local-acceptance/dependencies.lock | 8 +- deploy/kubernetes/local-acceptance/k3d.yaml | 35 +- .../operations/local-isomorphic-acceptance.md | 25 +- scripts/acceptance/local-cluster.sh | 307 +++++++++++++++++- scripts/acceptance/local-control-plane.sh | 94 ++++++ scripts/acceptance/report.mjs | 46 ++- scripts/acceptance/run-local-acceptance.sh | 148 +++++++-- tests/acceptance/report-test.mjs | 17 +- 20 files changed, 970 insertions(+), 85 deletions(-) create mode 100644 apps/api/internal/auth/auth_store_unavailable_test.go create mode 100644 scripts/acceptance/local-control-plane.sh diff --git a/apps/api/cmd/acceptance-bootstrap/main.go b/apps/api/cmd/acceptance-bootstrap/main.go index fea0a19..63ab1b3 100644 --- a/apps/api/cmd/acceptance-bootstrap/main.go +++ b/apps/api/cmd/acceptance-bootstrap/main.go @@ -119,6 +119,12 @@ func run(ctx context.Context, opts options) error { return err } defer database.Close() + if err := verifyLocalClusterMarker(ctx, database, opts.clusterID); err != nil { + return err + } + if err := resetPreviousLocalRun(ctx, database, opts); err != nil { + return err + } if err := verifyLocalDatabase(ctx, database, opts.clusterID); err != nil { return err } @@ -212,7 +218,15 @@ func run(ctx context.Context, opts options) error { if err != nil { return err } - if err := os.WriteFile(outputPath, append(payload, '\n'), 0o600); err != nil { + outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := outputFile.Write(append(payload, '\n')); err != nil { + _ = outputFile.Close() + return err + } + if err := outputFile.Close(); err != nil { return err } fmt.Printf( @@ -226,7 +240,60 @@ func run(ctx context.Context, opts options) error { return nil } +func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error { + mode, err := database.GetGatewayTrafficMode(ctx) + if err != nil { + return err + } + if mode.Mode == "live" { + return nil + } + if mode.Mode != "validation" || strings.TrimSpace(mode.RunID) == "" { + return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode) + } + run, err := database.GetAcceptanceRun(ctx, mode.RunID) + if err != nil { + return err + } + if run.Config["localClusterId"] != opts.clusterID { + return errors.New("refusing to replace an acceptance run owned by another cluster") + } + var outstanding int + if err := database.Pool().QueryRow(ctx, ` +SELECT count(*) +FROM gateway_tasks +WHERE acceptance_run_id = $1::uuid + AND status IN ('queued','running')`, run.ID).Scan(&outstanding); err != nil { + return err + } + if outstanding != 0 { + return fmt.Errorf("refusing to replace local acceptance run with %d outstanding tasks", outstanding) + } + _, err = database.AbortAcceptanceRun(ctx, store.PromoteAcceptanceRunInput{ + RunID: mode.RunID, + Revision: mode.Revision, + ReleaseSHA: mode.ReleaseSHA, + APIImageDigest: mode.APIImageDigest, + WorkerImageDigest: mode.WorkerImageDigest, + }) + return err +} + func verifyLocalDatabase(ctx context.Context, database *store.Store, clusterID string) error { + if err := verifyLocalClusterMarker(ctx, database, clusterID); err != nil { + return err + } + mode, err := database.GetGatewayTrafficMode(ctx) + if err != nil { + return err + } + if mode.Mode != "live" { + return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode) + } + return nil +} + +func verifyLocalClusterMarker(ctx context.Context, database *store.Store, clusterID string) error { var marker string err := database.Pool().QueryRow(ctx, ` SELECT COALESCE(value->>'clusterId', '') @@ -238,13 +305,6 @@ WHERE setting_key = $1`, acceptancesnapshot.LocalClusterSettingKey).Scan(&marker if marker != clusterID { return errors.New("refusing bootstrap: local cluster marker mismatch") } - mode, err := database.GetGatewayTrafficMode(ctx) - if err != nil { - return err - } - if mode.Mode != "live" { - return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode) - } return nil } @@ -486,10 +546,8 @@ func privateOutputPath(path string) (string, error) { if err != nil { return "", err } - if info, err := os.Lstat(absolute); err == nil { - if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { - return "", errors.New("runtime output must be a regular non-symlink file") - } + if _, err := os.Lstat(absolute); err == nil { + return "", errors.New("runtime output already exists") } else if !os.IsNotExist(err) { return "", err } diff --git a/apps/api/cmd/acceptance-load/main.go b/apps/api/cmd/acceptance-load/main.go index cb8f27f..78060bc 100644 --- a/apps/api/cmd/acceptance-load/main.go +++ b/apps/api/cmd/acceptance-load/main.go @@ -59,15 +59,16 @@ type options struct { } type report struct { - SchemaVersion string `json:"schemaVersion"` - RunID string `json:"runId"` - Profile string `json:"profile"` - StartedAt time.Time `json:"startedAt"` - FinishedAt time.Time `json:"finishedAt"` - Passed bool `json:"passed"` - Phases []phaseReport `json:"phases"` - Failure string `json:"failure,omitempty"` - SecretSafe bool `json:"secretSafe"` + SchemaVersion string `json:"schemaVersion"` + RunID string `json:"runId"` + Profile string `json:"profile"` + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Passed bool `json:"passed"` + Phases []phaseReport `json:"phases"` + Failure string `json:"failure,omitempty"` + FailureOperation string `json:"failureOperation,omitempty"` + SecretSafe bool `json:"secretSafe"` } type phaseReport struct { @@ -101,6 +102,25 @@ type httpStatusError struct { Body string } +type operationError struct { + Operation string + Err error +} + +func (e *operationError) Error() string { return e.Operation + ": " + e.Err.Error() } +func (e *operationError) Unwrap() error { return e.Err } + +func withOperation(operation string, err error) error { + if err == nil { + return nil + } + var existing *operationError + if errors.As(err, &existing) { + return err + } + return &operationError{Operation: operation, Err: err} +} + func (e *httpStatusError) Error() string { return fmt.Sprintf("HTTP %d: %s", e.Status, e.Body) } @@ -137,11 +157,15 @@ func main() { result.Passed = runErr == nil if runErr != nil { result.Failure = redactError(runErr.Error(), opts) + var operationErr *operationError + if errors.As(runErr, &operationErr) { + result.FailureOperation = operationErr.Operation + } } payload, _ := json.MarshalIndent(result, "", " ") fmt.Println(string(payload)) if opts.reportPath != "" { - if err := os.WriteFile(opts.reportPath, append(payload, '\n'), 0o600); err != nil { + if err := writeReportExclusive(opts.reportPath, payload); err != nil { fmt.Fprintln(os.Stderr, "write acceptance report failed:", err) os.Exit(1) } @@ -151,6 +175,18 @@ func main() { } } +func writeReportExclusive(path string, payload []byte) error { + reportFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + if _, err := reportFile.Write(append(payload, '\n')); err != nil { + _ = reportFile.Close() + return err + } + return reportFile.Close() +} + func parseOptions() (options, error) { var profile string var reportPath string @@ -347,14 +383,15 @@ func runGemini( response, err = client.Do(req) if err == nil { if response.StatusCode != http.StatusOK { - err = responseStatusError(response) + err = withOperation("gemini_submit", responseStatusError(response)) } else { var size int64 var outputHash string size, outputHash, err = streamGeminiImageHash(response.Body) + err = withOperation("gemini_response", err) _ = response.Body.Close() if err == nil && expectedOutputBytes > 0 && (size != int64(expectedOutputBytes) || outputHash != expectedHash) { - err = fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash) + err = withOperation("gemini_response", fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash)) } if err == nil { mu.Lock() @@ -367,6 +404,7 @@ func runGemini( } } } + err = withOperation("gemini_submit", err) latency := time.Since(requestStarted) mu.Lock() latencies[index] = latency @@ -503,7 +541,7 @@ func runVideo( response, err = client.Do(req) if err == nil { if response.StatusCode != http.StatusAccepted { - err = responseStatusError(response) + err = withOperation("video_submit", responseStatusError(response)) } else { var accepted struct { TaskID string `json:"taskId"` @@ -517,6 +555,7 @@ func runVideo( } } } + err = withOperation("video_submit", err) mu.Lock() latencies[index] = time.Since(requestStarted) if !realUpstream && logicalIndex%4 == 0 { @@ -742,7 +781,7 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/ai/result/" + url.PathEscape(taskID) req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return err + return withOperation("video_poll", err) } opts.setHeaders(req, index, realUpstream) response, err := client.Do(req) @@ -758,20 +797,20 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI case "succeeded", "success": mediaURL := findMediaURL(payload) if mediaURL == "" { - return fmt.Errorf("video task %s succeeded without a media URL", taskID) + return withOperation("video_poll", fmt.Errorf("video task %s succeeded without a media URL", taskID)) } - return validateVideoAsset(ctx, client, opts, mediaURL, index) + return withOperation("video_download", validateVideoAsset(ctx, client, opts, mediaURL, index)) case "failed", "cancelled", "canceled": - return fmt.Errorf("video task %s finished with status %s", taskID, status) + return withOperation("video_poll", fmt.Errorf("video task %s finished with status %s", taskID, status)) } } } if err != nil { - return err + return withOperation("video_poll", err) } select { case <-ctx.Done(): - return ctx.Err() + return withOperation("video_poll", ctx.Err()) case <-ticker.C: } } diff --git a/apps/api/cmd/acceptance-load/main_test.go b/apps/api/cmd/acceptance-load/main_test.go index bcb4432..0df750e 100644 --- a/apps/api/cmd/acceptance-load/main_test.go +++ b/apps/api/cmd/acceptance-load/main_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "net/http" "net/http/httptest" @@ -212,6 +213,35 @@ func TestAcceptanceReportErrorRedactsSecretsAndURLs(t *testing.T) { } } +func TestAcceptanceReportCannotOverwriteExistingRunArtifact(t *testing.T) { + path := filepath.Join(t.TempDir(), "load-report.json") + if err := writeReportExclusive(path, []byte(`{"runId":"first"}`)); err != nil { + t.Fatalf("write first report: %v", err) + } + if err := writeReportExclusive(path, []byte(`{"runId":"second"}`)); err == nil { + t.Fatal("second report unexpectedly overwrote the first report") + } + payload, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read report: %v", err) + } + if string(payload) != "{\"runId\":\"first\"}\n" { + t.Fatalf("report changed after rejected overwrite: %s", payload) + } +} + +func TestAcceptanceFailurePreservesOperationAndHTTPStatus(t *testing.T) { + err := withOperation("video_poll", &httpStatusError{Status: http.StatusUnauthorized, Body: "unauthorized"}) + var operationErr *operationError + if !errors.As(err, &operationErr) || operationErr.Operation != "video_poll" { + t.Fatalf("operation error=%#v", operationErr) + } + var statusErr *httpStatusError + if !errors.As(err, &statusErr) || statusErr.Status != http.StatusUnauthorized { + t.Fatalf("HTTP status error=%#v", statusErr) + } +} + func TestAcceptanceGatewayTLSNameSetsHostHeader(t *testing.T) { request := httptest.NewRequest(http.MethodGet, "https://127.0.0.1/api/v1/healthz", nil) opts := options{ diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index 7e6efcd..99853fd 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -5443,6 +5443,12 @@ "schema": { "$ref": "#/definitions/httpapi.EasyAIGeneratedResponse" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } } } } @@ -6323,6 +6329,12 @@ "schema": { "$ref": "#/definitions/httpapi.VolcesErrorEnvelope" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.VolcesErrorEnvelope" + } } } } @@ -7405,6 +7417,12 @@ "schema": { "$ref": "#/definitions/httpapi.GeminiErrorEnvelope" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.GeminiErrorEnvelope" + } } } } @@ -7478,6 +7496,12 @@ "schema": { "$ref": "#/definitions/httpapi.GeminiErrorEnvelope" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.GeminiErrorEnvelope" + } } } } @@ -7578,6 +7602,12 @@ "$ref": "#/definitions/httpapi.ErrorEnvelope" } }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, "504": { "description": "Gateway Timeout", "schema": { @@ -8446,6 +8476,12 @@ "$ref": "#/definitions/httpapi.ErrorEnvelope" } }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, "504": { "description": "Gateway Timeout", "schema": { @@ -8551,6 +8587,12 @@ "$ref": "#/definitions/httpapi.ErrorEnvelope" } }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, "504": { "description": "Gateway Timeout", "schema": { @@ -8999,6 +9041,12 @@ "$ref": "#/definitions/httpapi.ErrorEnvelope" } }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, "504": { "description": "Gateway Timeout", "schema": { @@ -9324,6 +9372,12 @@ "$ref": "#/definitions/httpapi.ErrorEnvelope" } }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, "504": { "description": "Gateway Timeout", "schema": { diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index bfa4e03..fdca05a 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -7680,6 +7680,10 @@ paths: description: Gone schema: $ref: '#/definitions/httpapi.EasyAIGeneratedResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' security: - BearerAuth: [] summary: 查询 server-main EasyAIClient 兼容任务结果 @@ -8248,6 +8252,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.VolcesErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.VolcesErrorEnvelope' security: - BearerAuth: [] summary: 创建火山内容生成任务 @@ -8945,6 +8953,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.GeminiErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.GeminiErrorEnvelope' security: - BearerAuth: [] summary: Gemini generateContent 兼容接口 @@ -8993,6 +9005,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.GeminiErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.GeminiErrorEnvelope' security: - BearerAuth: [] summary: Gemini generateContent 兼容接口 @@ -9058,6 +9074,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' "504": description: Gateway Timeout schema: @@ -9623,6 +9643,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' "504": description: Gateway Timeout schema: @@ -9692,6 +9716,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' "504": description: Gateway Timeout schema: @@ -9980,6 +10008,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' "504": description: Gateway Timeout schema: @@ -10192,6 +10224,10 @@ paths: description: Bad Gateway schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' "504": description: Gateway Timeout schema: diff --git a/apps/api/internal/auth/auth.go b/apps/api/internal/auth/auth.go index 3f477d3..d4204ca 100644 --- a/apps/api/internal/auth/auth.go +++ b/apps/api/internal/auth/auth.go @@ -69,9 +69,10 @@ const userContextKey contextKey = "easyai-auth-user" var ErrUnauthorized = errors.New("unauthorized") type RequestAuthError struct { - Status int - Code string - Message string + Status int + Code string + Message string + RetryAfterSeconds int } func (e *RequestAuthError) Error() string { return e.Code } @@ -80,6 +81,12 @@ func NewRequestAuthError(status int, code, message string) error { return &RequestAuthError{Status: status, Code: code, Message: message} } +func NewRetryableRequestAuthError(status int, code, message string, retryAfterSeconds int) error { + return &RequestAuthError{ + Status: status, Code: code, Message: message, RetryAfterSeconds: retryAfterSeconds, + } +} + type Authenticator struct { JWTSecret string ServerMainBaseURL string @@ -127,6 +134,9 @@ func (a *Authenticator) Require(permission Permission, next http.Handler) http.H } var requestError *RequestAuthError if errors.As(err, &requestError) { + if requestError.RetryAfterSeconds > 0 { + w.Header().Set("Retry-After", strconv.Itoa(requestError.RetryAfterSeconds)) + } w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Cache-Control", "no-store") w.WriteHeader(requestError.Status) @@ -318,7 +328,12 @@ func (a *Authenticator) verifyAPIKey(ctx context.Context, apiKey string) (*User, return user, nil } if !errors.Is(err, ErrUnauthorized) { - return nil, err + return nil, NewRetryableRequestAuthError( + http.StatusServiceUnavailable, + "AUTH_STORE_UNAVAILABLE", + "authentication service temporarily unavailable", + 2, + ) } if strings.HasPrefix(apiKey, "sk-gw-") { return nil, ErrUnauthorized diff --git a/apps/api/internal/auth/auth_store_unavailable_test.go b/apps/api/internal/auth/auth_store_unavailable_test.go new file mode 100644 index 0000000..5bc1740 --- /dev/null +++ b/apps/api/internal/auth/auth_store_unavailable_test.go @@ -0,0 +1,31 @@ +package auth + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +func TestAPIKeyStoreFailureReturnsRetryableServiceUnavailable(t *testing.T) { + authenticator := New("test-secret", "", "") + authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*User, error) { + return nil, errors.New("database unavailable") + } + handler := authenticator.Require(PermissionBasic, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("unavailable authentication store must not call the protected handler") + })) + request := httptest.NewRequest(http.MethodGet, "/protected", nil) + request.Header.Set("Authorization", "Bearer sk-gw-local") + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Retry-After") != "2" { + t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After")) + } +} diff --git a/apps/api/internal/httpapi/compat_protocol.go b/apps/api/internal/httpapi/compat_protocol.go index eb82700..0d61785 100644 --- a/apps/api/internal/httpapi/compat_protocol.go +++ b/apps/api/internal/httpapi/compat_protocol.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "net/http" + "strconv" "strings" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" @@ -19,6 +20,9 @@ func (s *Server) requireProtocolUser(protocol string, next http.Handler) http.Ha var requestErr *auth.RequestAuthError if errors.As(err, &requestErr) { status, code, message = requestErr.Status, requestErr.Code, requestErr.Message + if requestErr.RetryAfterSeconds > 0 { + w.Header().Set("Retry-After", strconv.Itoa(requestErr.RetryAfterSeconds)) + } } writeProtocolError(w, protocol, status, message, nil, code) return diff --git a/apps/api/internal/httpapi/compat_protocol_test.go b/apps/api/internal/httpapi/compat_protocol_test.go index 6d45dbb..a49f6fc 100644 --- a/apps/api/internal/httpapi/compat_protocol_test.go +++ b/apps/api/internal/httpapi/compat_protocol_test.go @@ -3,6 +3,7 @@ package httpapi import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "strings" @@ -13,6 +14,39 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) +func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) { + authenticator := auth.New("test-secret", "", "") + authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*auth.User, error) { + return nil, errors.New("database unavailable") + } + server := &Server{auth: authenticator} + handler := server.requireProtocolUser(clients.ProtocolGeminiGenerateContent, http.HandlerFunc( + func(http.ResponseWriter, *http.Request) { + t.Fatal("unavailable authentication store must not call the protocol handler") + }, + )) + request := httptest.NewRequest(http.MethodPost, "/v1beta/models/test:generateContent", nil) + request.Header.Set("Authorization", "Bearer sk-gw-local") + recorder := httptest.NewRecorder() + + handler.ServeHTTP(recorder, request) + + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String()) + } + if recorder.Header().Get("Retry-After") != "2" { + t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After")) + } + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode response: %v", err) + } + errorBody := requireObject(t, body["error"]) + if errorBody["code"] != float64(http.StatusServiceUnavailable) || errorBody["status"] != "UNAVAILABLE" { + t.Fatalf("unexpected Gemini unavailable response: %+v", body) + } +} + func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) { tests := []struct { name string diff --git a/apps/api/internal/httpapi/gemini_compat.go b/apps/api/internal/httpapi/gemini_compat.go index 6a93cd1..85bb1ee 100644 --- a/apps/api/internal/httpapi/gemini_compat.go +++ b/apps/api/internal/httpapi/gemini_compat.go @@ -106,6 +106,7 @@ func geminiGenerateContentModelFromPath(prefix string, requestPath string) (stri // @Failure 401 {object} GeminiErrorEnvelope // @Failure 403 {object} GeminiErrorEnvelope // @Failure 502 {object} GeminiErrorEnvelope +// @Failure 503 {object} GeminiErrorEnvelope // @Router /api/v1/models/{model}:generateContent [post] // @Router /api/v1/models/{model}:streamGenerateContent [post] func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 09059e0..c82a9a7 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -1420,6 +1420,7 @@ func openAIImagesDoc() {} // @Failure 429 {object} ErrorEnvelope // @Failure 504 {object} ErrorEnvelope // @Failure 502 {object} ErrorEnvelope +// @Failure 503 {object} ErrorEnvelope // @Router /api/v1/videos/generations [post] // @Router /api/v1/song/generations [post] // @Router /api/v1/music/generations [post] diff --git a/apps/api/internal/httpapi/volces_compat_handlers.go b/apps/api/internal/httpapi/volces_compat_handlers.go index 46d3b47..d1f5ad4 100644 --- a/apps/api/internal/httpapi/volces_compat_handlers.go +++ b/apps/api/internal/httpapi/volces_compat_handlers.go @@ -28,6 +28,7 @@ const volcesContentsCompatibilityMarker = "volces_contents_generations_v3" // @Failure 401 {object} VolcesErrorEnvelope // @Failure 429 {object} VolcesErrorEnvelope // @Failure 502 {object} VolcesErrorEnvelope +// @Failure 503 {object} VolcesErrorEnvelope // @Router /api/v1/contents/generations/tasks [post] func (s *Server) createVolcesContentsGenerationTask(w http.ResponseWriter, r *http.Request) { user, ok := auth.UserFromContext(r.Context()) @@ -186,6 +187,7 @@ func (s *Server) createLegacyVolcesVideoGeneration(w http.ResponseWriter, r *htt // @Success 200 {object} EasyAIGeneratedResponse // @Failure 404 {object} EasyAIGeneratedResponse // @Failure 410 {object} EasyAIGeneratedResponse +// @Failure 503 {object} ErrorEnvelope // @Router /api/v1/ai/result/{taskID} [get] func (s *Server) getEasyAITaskResult(w http.ResponseWriter, r *http.Request) { user, ok := auth.UserFromContext(r.Context()) diff --git a/deploy/kubernetes/local-acceptance/dependencies.lock b/deploy/kubernetes/local-acceptance/dependencies.lock index a252736..ab84853 100644 --- a/deploy/kubernetes/local-acceptance/dependencies.lock +++ b/deploy/kubernetes/local-acceptance/dependencies.lock @@ -1,8 +1,14 @@ # Local acceptance dependency lock. Checksums are lowercase SHA-256. K3D_VERSION=v5.9.0 +CRANE_VERSION=v0.21.7 K3D_DARWIN_ARM64_SHA256=fe106541d5d0a3f18debcd4d432a16f8c0ce3e6ddc06f8fbb6f696a122313e00 K3D_DARWIN_AMD64_SHA256=b4aabc37534f95b9c764e7823f2df923f50d57600837aa60a06266cce47db732 -K3S_IMAGE=rancher/k3s:v1.36.2-k3s1 +K3S_IMAGE=rancher/k3s:v1.36.2-k3s1@sha256:6a47cea22c4b834d4ba72c89d291696b79ebe406251f90b446e4dff03513dd87 CNPG_VERSION=1.29.1 CNPG_MANIFEST_URL=https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.29/releases/cnpg-1.29.1.yaml CNPG_MANIFEST_SHA256=e0c5ff41bf5b01c0775bf225929b8423d543cd23e9742e4274fe7de8499cf93a +CNPG_CONTROLLER_IMAGE=ghcr.io/cloudnative-pg/cloudnative-pg:1.29.1@sha256:0dfff19ba7b52ca25851a1010028b6940fff2e233290465af1cfb08a5f3f4661 +CNPG_POSTGRES_IMAGE=ghcr.io/cloudnative-pg/postgresql:18.4-standard-trixie@sha256:4587df73024408f5b2be9b4dd6ba2ccee8c9e5dc0c9a87c274c292291cc8a68c +K3S_COREDNS_IMAGE=rancher/mirrored-coredns-coredns:1.14.4@sha256:3e98f280fd601b37411c5fb7075fd9f337833c480f1644970b727ae0af067782 +K3S_METRICS_SERVER_IMAGE=rancher/mirrored-metrics-server:v0.8.1@sha256:b2d2efaf5ac3b366ed0f839d2412a2c4279d4fc2a2a733f12c52133faed36c41 +K3S_LOCAL_PATH_IMAGE=rancher/local-path-provisioner:v0.0.36@sha256:1eba82e9c386038b4af6d69cca7519fac738c28c42735ed48ce70c882ad0d80f diff --git a/deploy/kubernetes/local-acceptance/k3d.yaml b/deploy/kubernetes/local-acceptance/k3d.yaml index 5f2d695..4e78b58 100644 --- a/deploy/kubernetes/local-acceptance/k3d.yaml +++ b/deploy/kubernetes/local-acceptance/k3d.yaml @@ -4,7 +4,8 @@ metadata: name: easyai-acceptance-local servers: 3 agents: 0 -image: rancher/k3s:v1.36.2-k3s1 +image: EASYAI_ACCEPTANCE_K3S_IMAGE +subnet: EASYAI_ACCEPTANCE_CLUSTER_SUBNET kubeAPI: host: 127.0.0.1 hostPort: "16550" @@ -19,6 +20,21 @@ volumes: - volume: EASYAI_ACCEPTANCE_MEDIA_DIR:/var/lib/easyai-acceptance/media nodeFilters: - server:* + - volume: easyai-acceptance-local-server-0-etcd:/var/lib/rancher/k3s/server/db + nodeFilters: + - server:0 + - volume: easyai-acceptance-local-server-1-etcd:/var/lib/rancher/k3s/server/db + nodeFilters: + - server:1 + - volume: easyai-acceptance-local-server-2-etcd:/var/lib/rancher/k3s/server/db + nodeFilters: + - server:2 + - volume: easyai-acceptance-local-server-0-storage:/var/lib/rancher/k3s/storage + nodeFilters: + - server:0 + - volume: easyai-acceptance-local-server-1-storage:/var/lib/rancher/k3s/storage + nodeFilters: + - server:1 options: k3s: extraArgs: @@ -48,3 +64,20 @@ options: - arg: --node-taint=easyai.io/control-plane-only=true:NoSchedule nodeFilters: - server:2 + - arg: --kubelet-arg=system-reserved=cpu=EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU,memory=EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY + nodeFilters: + - server:0 + - server:1 + - arg: --kubelet-arg=kube-reserved=cpu=250m,memory=512Mi + nodeFilters: + - server:0 + - server:1 + - arg: --kubelet-arg=system-reserved=cpu=EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU,memory=EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY + nodeFilters: + - server:2 + - arg: --kubelet-arg=kube-reserved=cpu=250m,memory=512Mi + nodeFilters: + - server:2 + - arg: --kubelet-arg=eviction-hard=memory.available<512Mi,nodefs.available<10% + nodeFilters: + - server:* diff --git a/docs/operations/local-isomorphic-acceptance.md b/docs/operations/local-isomorphic-acceptance.md index e784cb0..403d165 100644 --- a/docs/operations/local-isomorphic-acceptance.md +++ b/docs/operations/local-isomorphic-acceptance.md @@ -25,11 +25,23 @@ | server-1 / hongkong | 4 CPU / 8 GiB | API、Worker、PostgreSQL | | server-2 / los-angeles | 2 CPU / 4 GiB | 控制面和 etcd，`NoSchedule` | -Docker Desktop 必须配置至少 24 GiB 内存；脚本只检查，不修改 Docker 设置。k3d、K3s 和 +Docker Desktop 必须配置至少 24 GiB 内存，宿主工作盘必须至少保留 30 GiB 可用空间； +脚本只检查，不修改 Docker 或宿主设置。依赖预载后会先用锁定 K3s 镜像执行 10 秒容器 +启动探针，应用镜像构建后再用本次验收的最小镜像复检，确保 Docker daemon 不只是 API +可读、而是真的能创建容器，再进入 K3d。 +k3d、K3s 和 CNPG 版本及 SHA-256 位于 `deploy/kubernetes/local-acceptance/dependencies.lock`。高并发阶段使用宿主原生架构镜像， 最后才导入 release manifest 中精确的 `linux/amd64@sha256` 制品做启动、迁移和媒体冒烟。 +K3s 启动时会通过 Kubelet `system-reserved`/`kube-reserved` 将两个业务节点的可调度资源 +固定为 3 CPU/6656 MiB，将见证节点固定为 1 CPU/2560 MiB；随后 Docker 硬上限分别设置为 +4C/8GiB、4C/8GiB、2C/4GiB。API CPU 为 250m/750m，Worker 为 750m/1500m，Worker +内存仍为 1536Mi/2Gi。etcd 与 PostgreSQL `local-path` 使用不同的命名卷，避免和业务媒体 +目录共用 Docker writable layer。依赖镜像会在部署应用前按锁定 digest 预拉取并导入；若 +Docker daemon 的系统代理不可用，则使用锁定版本的 `crane` 按同一 digest 拉取 OCI 内容， +再导入本地 Docker/K3s，不会退化为未校验 Tag。 + ```bash scripts/acceptance/local-cluster.sh install-tools scripts/acceptance/local-cluster.sh preflight @@ -70,6 +82,10 @@ scripts/acceptance/local-cluster.sh up \ scripts/acceptance/run-local-acceptance.sh quick +# P24 基线应使用三个不同 Run ID 连续通过 +scripts/acceptance/run-local-acceptance.sh quick +scripts/acceptance/run-local-acceptance.sh quick + scripts/acceptance/run-local-acceptance.sh full \ --release-manifest dist/releases/<完整SHA>.json ``` @@ -85,6 +101,13 @@ scripts/acceptance/run-local-acceptance.sh full \ 7. 80% 两小时混合流量和 120% 十分钟主动限流。 8. 精确 amd64 release 镜像的迁移、启动与媒体冒烟。 +每次 `quick`、`full` 或独立 `artifact-smoke` 都会先生成新的 Run ID；前一 Run 只有在队列 +和 running 已归零后才会被本地安全终止。负载报告使用独占创建，已有文件不会被覆盖。 +负载期间每 2 秒核对三个 K3s server 的容器 ID、restartCount、启动时间、IP、Node UID、 +API readiness；任一 server 重启立即以 `local_control_plane_restarted` 终止。etcd 出现严重 +心跳、fdatasync、ReadIndex 或请求超时则以 `local_etcd_latency` 阻断。本地 P24 基线先把 +6K 图片归一化并发固定为 1，确认稳定后才允许单独测试更高媒体并发。 + 网络故障只能作用于带 `easyai.io/environment=local-acceptance` 标签的代理 Pod： ```bash diff --git a/scripts/acceptance/local-cluster.sh b/scripts/acceptance/local-cluster.sh index 77e6659..5463389 100755 --- a/scripts/acceptance/local-cluster.sh +++ b/scripts/acceptance/local-cluster.sh @@ -12,6 +12,11 @@ media_root="$private_root/media" cluster_name=easyai-acceptance-local context="k3d-$cluster_name" namespace=easyai +cluster_network="k3d-$cluster_name" +cluster_subnet=172.30.0.0/16 +identity_file="$state_root/control-plane-identity.json" +# shellcheck source=scripts/acceptance/local-control-plane.sh +source "$script_dir/local-control-plane.sh" usage() { cat <<'EOF' @@ -19,11 +24,13 @@ Usage: scripts/acceptance/local-cluster.sh install-tools scripts/acceptance/local-cluster.sh preflight scripts/acceptance/local-cluster.sh up --snapshot + scripts/acceptance/local-cluster.sh new-run scripts/acceptance/local-cluster.sh status scripts/acceptance/local-cluster.sh down --confirm -The full cluster requires Docker Desktop memory >= 24 GiB. Failed `up` runs are -kept intact for diagnosis. `down` is the only destructive lifecycle action. +The full cluster requires Docker Desktop memory >= 24 GiB and host free disk +>= 30 GiB. Failed `up` runs are kept intact for diagnosis. `down` is the only +destructive lifecycle action. EOF } @@ -34,7 +41,9 @@ load_lock() { } # shellcheck source=/dev/null source "$lock_file" - : "${K3D_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}" + : "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}" + : "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}" + : "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}" } require_command() { @@ -60,6 +69,7 @@ install_tools() { load_lock require_command curl require_command shasum + require_command go install -d -m 0700 "$tools_root" local os arch checksum url temporary os=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -91,7 +101,9 @@ install_tools() { } chmod 0555 "$temporary" mv "$temporary" "$tools_root/k3d" - echo "local_acceptance_tools=PASS k3d=$K3D_VERSION path=$tools_root/k3d" + GOBIN="$tools_root" go install "github.com/google/go-containerregistry/cmd/crane@$CRANE_VERSION" + chmod 0555 "$tools_root/crane" + echo "local_acceptance_tools=PASS k3d=$K3D_VERSION crane=$CRANE_VERSION path=$tools_root" } k3d_binary() { @@ -106,6 +118,10 @@ docker_memory_bytes() { docker info --format '{{.MemTotal}}' } +host_available_disk_bytes() { + df -Pk "$repository_root" | awk 'NR == 2 {printf "%.0f\n", $4 * 1024}' +} + preflight() { load_lock require_command docker @@ -117,7 +133,7 @@ preflight() { require_command go require_command node require_command curl - local k3d memory_bytes architecture + local k3d memory_bytes architecture cpu_count available_disk_bytes available_disk_gib k3d=$(k3d_binary) || { echo 'k3d is missing; run install-tools' >&2 exit 1 @@ -126,6 +142,10 @@ preflight() { echo "k3d must be pinned to $K3D_VERSION" >&2 exit 1 } + if [[ ! -x $tools_root/crane ]] || ! "$tools_root/crane" version | grep -Fq "${CRANE_VERSION#v}"; then + echo "crane must be pinned to $CRANE_VERSION; run install-tools" >&2 + exit 1 + fi docker info >/dev/null memory_bytes=$(docker_memory_bytes) [[ $memory_bytes =~ ^[0-9]+$ && $memory_bytes -ge 25769803776 ]] || { @@ -139,7 +159,18 @@ preflight() { exit 1 } docker buildx version >/dev/null - echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes architecture=$architecture k3d=$K3D_VERSION" + cpu_count=$(docker info --format '{{.NCPU}}') + [[ $cpu_count =~ ^[0-9]+$ && $cpu_count -ge 10 ]] || { + echo "Docker Desktop CPUs=${cpu_count:-unknown}; local acceptance requires at least 10" >&2 + exit 1 + } + available_disk_bytes=$(host_available_disk_bytes) + [[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || { + available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}') + echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2 + exit 1 + } + echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION" } ensure_private_material() { @@ -185,9 +216,174 @@ generate_tls() { } render_k3d_config() { - local output=$1 + local output=$1 host_cpu host_memory_bytes host_memory_mib + local site_system_cpu witness_system_cpu site_system_memory witness_system_memory [[ $media_root != *'|'* ]] - sed "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" "$manifest_root/k3d.yaml" >"$output" + host_cpu=$(docker info --format '{{.NCPU}}') + host_memory_bytes=$(docker_memory_bytes) + host_memory_mib=$((host_memory_bytes / 1024 / 1024)) + site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m" + witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m" + site_system_memory="$((host_memory_mib - 8192 + 1024))Mi" + witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi" + sed \ + -e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \ + -e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \ + -e "s|EASYAI_ACCEPTANCE_CLUSTER_SUBNET|$cluster_subnet|g" \ + -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_CPU|$site_system_cpu|g" \ + -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_CPU|$witness_system_cpu|g" \ + -e "s|EASYAI_ACCEPTANCE_SITE_SYSTEM_MEMORY|$site_system_memory|g" \ + -e "s|EASYAI_ACCEPTANCE_WITNESS_SYSTEM_MEMORY|$witness_system_memory|g" \ + "$manifest_root/k3d.yaml" >"$output" +} + +cluster_volumes() { + printf '%s\n' \ + easyai-acceptance-local-server-0-etcd \ + easyai-acceptance-local-server-1-etcd \ + easyai-acceptance-local-server-2-etcd \ + easyai-acceptance-local-server-0-storage \ + easyai-acceptance-local-server-1-storage +} + +create_cluster_volumes() { + local volume + while read -r volume; do + if docker volume inspect "$volume" >/dev/null 2>&1; then + echo "stale local acceptance volume exists: $volume; run explicit down --confirm" >&2 + return 1 + fi + done < <(cluster_volumes) + while read -r volume; do + docker volume create --label easyai.io/environment=local-acceptance "$volume" >/dev/null + done < <(cluster_volumes) +} + +remove_cluster_volumes() { + local volume + while read -r volume; do + docker volume rm "$volume" >/dev/null 2>&1 || true + done < <(cluster_volumes) +} + +reconcile_cluster_network() { + docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0 + local actual_subnet attached_containers + actual_subnet=$(docker network inspect "$cluster_network" --format '{{(index .IPAM.Config 0).Subnet}}') + attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}') + if [[ $actual_subnet == "$cluster_subnet" ]]; then + return 0 + fi + if ((attached_containers != 0)); then + echo "stale local acceptance network has active containers: network=$cluster_network expected_subnet=$cluster_subnet actual_subnet=$actual_subnet" >&2 + return 1 + fi + docker network rm "$cluster_network" >/dev/null +} + +remove_cluster_network() { + docker network inspect "$cluster_network" >/dev/null 2>&1 || return 0 + local attached_containers + attached_containers=$(docker network inspect "$cluster_network" --format '{{len .Containers}}') + if ((attached_containers != 0)); then + echo "refusing to remove local acceptance network with active containers: network=$cluster_network containers=$attached_containers" >&2 + return 1 + fi + docker network rm "$cluster_network" >/dev/null +} + +pull_dependency_images() { + local image + for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \ + "$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do + pull_dependency_image "$image" + done +} + +docker_pull_with_deadline() { + local image=$1 pid deadline + docker pull "$image" >/dev/null 2>&1 & + pid=$! + deadline=$((SECONDS + 10)) + while kill -0 "$pid" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + return 1 + fi + sleep 1 + done + wait "$pid" +} + +docker_start_with_deadline() { + local container=$1 pid deadline + docker start "$container" >/dev/null 2>&1 & + pid=$! + deadline=$((SECONDS + 10)) + while kill -0 "$pid" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + kill "$pid" >/dev/null 2>&1 || true + wait "$pid" >/dev/null 2>&1 || true + return 1 + fi + sleep 1 + done + wait "$pid" +} + +verify_docker_container_start() { + local image=$1 container="easyai-acceptance-start-probe-$$" exit_code + docker create --name "$container" --network none --entrypoint /bin/true "$image" >/dev/null + if ! docker_start_with_deadline "$container"; then + docker rm -f "$container" >/dev/null 2>&1 || true + echo 'gate=local_docker_container_start_unavailable Docker cannot start a minimal acceptance container within 10 seconds' >&2 + return 1 + fi + exit_code=$(docker inspect "$container" --format '{{.State.ExitCode}}') + docker rm -f "$container" >/dev/null + [[ $exit_code == 0 ]] || { + echo "gate=local_docker_container_start_failed minimal acceptance container exit_code=$exit_code" >&2 + return 1 + } +} + +pull_dependency_image() { + local image=$1 name_tag repository digest cache_tag native_arch archive source_tag + name_tag=${image%%@*} + repository=${name_tag%:*} + digest=${image##*@sha256:} + [[ $digest =~ ^[0-9a-f]{64}$ ]] + cache_tag="$repository:easyai-lock-${digest:0:20}" + if docker image inspect "$image" >/dev/null 2>&1; then + docker tag "$image" "$name_tag" + docker tag "$image" "$cache_tag" + return + fi + if docker image inspect "$cache_tag" >/dev/null 2>&1; then + docker tag "$cache_tag" "$name_tag" + return + fi + if docker_pull_with_deadline "$image"; then + docker tag "$image" "$name_tag" + docker tag "$name_tag" "$cache_tag" + return + fi + native_arch=$(docker info --format '{{.Architecture}}') + case $native_arch in + aarch64|arm64) native_arch=arm64 ;; + x86_64) native_arch=amd64 ;; + *) return 1 ;; + esac + archive="$state_root/dependency-image.tar" + [[ ! -L $archive ]] + umask 077 + "$tools_root/crane" pull --platform "linux/$native_arch" "$image" "$archive" + docker load -i "$archive" >/dev/null + : >"$archive" + source_tag="$repository:i-was-a-digest" + docker tag "$source_tag" "$name_tag" + docker tag "$source_tag" "$cache_tag" } wait_rollout() { @@ -430,6 +626,15 @@ render_and_apply_application() { done kubectl --context "$context" -n "$namespace" scale \ deployment/easyai-web-ningbo deployment/easyai-web-hongkong --replicas=0 >/dev/null + kubectl --context "$context" -n "$namespace" set resources deployment/easyai-api-ningbo \ + deployment/easyai-api-hongkong --containers=api \ + --requests=cpu=250m,memory=512Mi --limits=cpu=750m,memory=2Gi >/dev/null + kubectl --context "$context" -n "$namespace" set resources deployment/easyai-worker-ningbo \ + deployment/easyai-worker-hongkong --containers=worker \ + --requests=cpu=750m,memory=1536Mi --limits=cpu=1500m,memory=2Gi >/dev/null + kubectl --context "$context" -n "$namespace" set env deployment/easyai-worker-ningbo \ + deployment/easyai-worker-hongkong \ + AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null } bootstrap_runtime() { @@ -439,7 +644,7 @@ bootstrap_runtime() { local snapshot_hash release_sha runtime_file snapshot_hash=$(jq -r '.snapshotSha256' "$snapshot") release_sha=$(git -C "$repository_root" rev-parse HEAD) - runtime_file="$state_root/runtime-$snapshot_hash.json" + runtime_file="$state_root/runtime-$snapshot_hash-$(date -u '+%Y%m%dT%H%M%SZ')-$$.json" kubectl --context "$context" -n "$namespace" delete pod easyai-local-bootstrap \ --ignore-not-found --wait=true >/dev/null cat </dev/null @@ -561,6 +766,8 @@ up_cluster() { api_image="easyai-acceptance-api:$release_sha" web_image="easyai-acceptance-web:$release_sha" netem_image="easyai-acceptance-netem:$release_sha" + pull_dependency_images + verify_docker_container_start "${K3S_IMAGE%%@*}" docker buildx build --load --platform "linux/$native_arch" --target api \ -t "$api_image" "$repository_root" docker buildx build --load --platform "linux/$native_arch" --target web \ @@ -569,7 +776,10 @@ up_cluster() { -t "$netem_image" "$repository_root" api_digest=$(docker image inspect "$api_image" --format '{{.Id}}') [[ $api_digest =~ ^sha256:[0-9a-f]{64}$ ]] + verify_docker_container_start "$netem_image" + reconcile_cluster_network + create_cluster_volumes config="$state_root/k3d.rendered.yaml" render_k3d_config "$config" EASYAI_ACCEPTANCE_MEDIA_DIR="$media_root" "$k3d" cluster create --config "$config" @@ -577,7 +787,14 @@ up_cluster() { docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-0" >/dev/null docker update --cpus 4 --memory 8g --memory-swap 8g "k3d-${cluster_name}-server-1" >/dev/null docker update --cpus 2 --memory 4g --memory-swap 4g "k3d-${cluster_name}-server-2" >/dev/null - "$k3d" image import -c "$cluster_name" "$api_image" "$web_image" "$netem_image" + verify_local_node_capacity "$context" || { + echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2 + exit 1 + } + "$k3d" image import -c "$cluster_name" \ + "$api_image" "$web_image" "$netem_image" \ + "${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \ + "${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}" local cnpg_manifest="$state_root/cnpg.yaml" curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest" @@ -590,7 +807,8 @@ up_cluster() { kubectl --context "$context" -n cnpg-system rollout status \ deployment/cnpg-controller-manager --timeout=10m apply_runtime_secrets - kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/database.yaml" >/dev/null + sed "s|imageName: .*|imageName: ${CNPG_POSTGRES_IMAGE%%@*}|" "$manifest_root/database.yaml" | + kubectl --context "$context" -n "$namespace" apply -f - >/dev/null kubectl --context "$context" -n "$namespace" wait --for=condition=Ready \ cluster/easyai-postgres --timeout=15m >/dev/null run_migrations "$api_image" @@ -619,9 +837,58 @@ up_cluster() { bootstrap_runtime "$api_image" "$api_digest" "$snapshot" wait_internal_gateway "$script_dir/network-fault.sh" baseline + sleep 1 + record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" echo "local_acceptance_cluster=PASS context=$context gateways=https://127.0.0.1:18443,https://127.0.0.1:19443 ca_file=$state_root/ca.crt" } +new_run() { + load_lock + preflight + [[ -z $(git -C "$repository_root" status --short) ]] || { + echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2 + exit 1 + } + if ! private_file "$state_root/current-runtime-path" || ! private_file "$state_root/snapshot.json"; then + echo 'local acceptance runtime or snapshot is unavailable' >&2 + exit 1 + fi + verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || { + echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild the cluster' >&2 + exit 1 + } + verify_local_node_capacity "$context" || { + echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2 + exit 1 + } + verify_local_apiserver_ready "$context" || { + echo 'gate=local_control_plane_unavailable local Kubernetes API is not ready' >&2 + exit 1 + } + local previous_runtime api_image api_digest previous_release current_release + previous_runtime=$(<"$state_root/current-runtime-path") + private_file "$previous_runtime" || { + echo 'local acceptance current runtime is unavailable' >&2 + exit 1 + } + previous_release=$(jq -r '.releaseSha' "$previous_runtime") + current_release=$(git -C "$repository_root" rev-parse HEAD) + [[ $previous_release == "$current_release" ]] || { + echo 'local acceptance source changed after cluster creation; rebuild the cluster' >&2 + exit 1 + } + api_digest=$(jq -r '.apiImageDigest' "$previous_runtime") + api_image=$(kubectl --context "$context" -n "$namespace" get deployment easyai-api-ningbo \ + -o 'jsonpath={.spec.template.spec.containers[?(@.name=="api")].image}') + [[ -n $api_image && $api_digest =~ ^sha256:[0-9a-f]{64}$ ]] + bootstrap_runtime "$api_image" "$api_digest" "$state_root/snapshot.json" + verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || { + echo 'gate=local_control_plane_restarted local K3s server changed while creating a fresh run' >&2 + exit 1 + } + echo "local_acceptance_new_run=PASS runtime_path=$state_root/current-runtime-path" +} + status_cluster() { local k3d k3d=$(k3d_binary) || { @@ -634,6 +901,14 @@ status_cluster() { } kubectl --context "$context" get nodes -o wide kubectl --context "$context" -n "$namespace" get cluster,pod,deployment + verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || { + echo 'gate=local_control_plane_restarted local K3s server identity changed; rebuild required' >&2 + exit 1 + } + verify_local_node_capacity "$context" || { + echo 'gate=local_node_capacity_mismatch local node allocatable capacity drifted' >&2 + exit 1 + } echo "local_acceptance_status=PASS context=$context" } @@ -644,7 +919,11 @@ down_cluster() { } local k3d k3d=$(k3d_binary) - "$k3d" cluster delete "$cluster_name" + if "$k3d" cluster list --no-headers | awk '{print $1}' | grep -Fxq "$cluster_name"; then + "$k3d" cluster delete "$cluster_name" + fi + remove_cluster_volumes + remove_cluster_network echo "local_acceptance_down=PASS preserved_private_root=$private_root" } @@ -661,6 +940,10 @@ case ${1:-} in shift up_cluster "$@" ;; + new-run) + [[ $# -eq 1 ]] || { usage >&2; exit 64; } + new_run + ;; status) [[ $# -eq 1 ]] || { usage >&2; exit 64; } status_cluster diff --git a/scripts/acceptance/local-control-plane.sh b/scripts/acceptance/local-control-plane.sh new file mode 100644 index 0000000..7bb8ad8 --- /dev/null +++ b/scripts/acceptance/local-control-plane.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Shared read-only guards for the local three-server K3s acceptance cluster. +# The caller owns shell options and failure reporting. + +capture_local_control_plane_identity() { + local context=$1 cluster_name=$2 containers nodes + containers=$(docker inspect \ + "k3d-${cluster_name}-server-0" \ + "k3d-${cluster_name}-server-1" \ + "k3d-${cluster_name}-server-2" | + jq -c --arg network "k3d-$cluster_name" '[.[] | { + name:(.Name | ltrimstr("/")), + id:.Id, + restartCount:.RestartCount, + startedAt:.State.StartedAt, + ip:.NetworkSettings.Networks[$network].IPAddress + }] | sort_by(.name)') || return 1 + nodes=$(kubectl --context "$context" get nodes -o json | + jq -c '[.items[] | { + name:.metadata.name, + uid:.metadata.uid, + site:.metadata.labels["easyai.io/site"], + cpu:.status.allocatable.cpu, + memory:.status.allocatable.memory + }] | sort_by(.name)') || return 1 + jq -cn --argjson containers "$containers" --argjson nodes "$nodes" \ + '{containers:$containers,nodes:$nodes}' +} + +record_local_control_plane_identity() { + local context=$1 cluster_name=$2 output=$3 current temporary + current=$(capture_local_control_plane_identity "$context" "$cluster_name") || return 1 + temporary="${output}.tmp.$$" + jq -cn \ + --arg recordedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ + --argjson current "$current" \ + '{schemaVersion:"local-control-plane-identity/v1",recordedAt:$recordedAt} + $current' \ + >"$temporary" + chmod 0600 "$temporary" + mv "$temporary" "$output" +} + +verify_local_control_plane_identity() { + local context=$1 cluster_name=$2 baseline=$3 expected current + [[ -f $baseline && ! -L $baseline ]] || return 1 + expected=$(jq -cS '{containers,nodes}' "$baseline") || return 1 + current=$(capture_local_control_plane_identity "$context" "$cluster_name" | + jq -cS '{containers,nodes}') || return 1 + [[ $current == "$expected" ]] +} + +verify_local_node_capacity() { + local context=$1 + kubectl --context "$context" get nodes -o json | jq -e ' + def cpu_m: + if endswith("m") then rtrimstr("m") | tonumber else tonumber * 1000 end; + def memory_mi: + if endswith("Ki") then rtrimstr("Ki") | tonumber / 1024 + elif endswith("Mi") then rtrimstr("Mi") | tonumber + elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024 + else 0 end; + ([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") | + select((.status.allocatable.cpu | cpu_m) == 3000 and + (.status.allocatable.memory | memory_mi) == 6656)] | length) == 2 and + ([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") | + select((.status.allocatable.cpu | cpu_m) == 1000 and + (.status.allocatable.memory | memory_mi) == 2560)] | length) == 1 and + ([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and + ([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0 + ' >/dev/null +} + +verify_local_apiserver_ready() { + local context=$1 + kubectl --context "$context" get --raw='/readyz?verbose' 2>/dev/null | + grep -Fq 'readyz check passed' +} + +verify_local_etcd_runtime_logs() { + local cluster_name=$1 baseline=$2 recorded_at name + [[ -f $baseline && ! -L $baseline ]] || return 1 + recorded_at=$(jq -r '.recordedAt' "$baseline") || return 1 + [[ $recorded_at == ????-??-??T*Z ]] + for name in \ + "k3d-${cluster_name}-server-0" \ + "k3d-${cluster_name}-server-1" \ + "k3d-${cluster_name}-server-2"; do + if docker logs --since "$recorded_at" "$name" 2>&1 | + grep -Eqi 'heartbeat interval is too long|apply request took too long|slow fdatasync|failed to check local etcd status|etcdserver: request timed out|waiting for ReadIndex response took too long'; then + return 1 + fi + done +} diff --git a/scripts/acceptance/report.mjs b/scripts/acceptance/report.mjs index d0deff7..cd48e0b 100755 --- a/scripts/acceptance/report.mjs +++ b/scripts/acceptance/report.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import { createHash } from 'node:crypto'; -import { lstat, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { lstat, mkdir, readFile, readdir, rename, unlink, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; const shaPattern = /^[0-9a-f]{40}$/; @@ -26,6 +26,29 @@ async function regularJSON(path) { return JSON.parse(await readFile(absolute, 'utf8')); } +async function writeNewJSON(path, value) { + const absolute = resolve(path); + await mkdir(dirname(absolute), { recursive: true, mode: 0o700 }); + await writeFile(absolute, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' }); +} + +async function replaceJSONAtomically(input, output, value) { + const inputPath = resolve(input); + const outputPath = resolve(output); + if (inputPath !== outputPath) { + await writeNewJSON(outputPath, value); + return; + } + const temporary = `${outputPath}.tmp-${process.pid}-${Date.now()}`; + try { + await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600, flag: 'wx' }); + await rename(temporary, outputPath); + } catch (error) { + await unlink(temporary).catch(() => {}); + throw error; + } +} + function assertSecretSafe(value, path = '$') { if (Array.isArray(value)) { value.forEach((item, index) => assertSecretSafe(item, `${path}[${index}]`)); @@ -85,6 +108,8 @@ async function loadReports(directory) { profile: report.profile, passed: report.passed, phases: report.phases, + failure: report.failure, + failureOperation: report.failureOperation, startedAt: report.startedAt, finishedAt: report.finishedAt }); @@ -150,8 +175,7 @@ async function buildLocal(values) { secretSafe: true }; validate(report); - await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 }); - await writeFile(resolve(values.output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + await writeNewJSON(values.output, report); process.stdout.write(`acceptance_report_local=PASS sha256=${createHash('sha256').update(JSON.stringify(report)).digest('hex')}\n`); } @@ -172,6 +196,10 @@ async function buildLocalPartial(values) { throw new Error('partial local report inputs have incompatible CAS identity'); } const completedLoadsPassed = loads.every((item) => item.passed === true); + const failureGate = values['failure-gate'] ?? 'local_execution_incomplete'; + if (!/^[a-z0-9_]+$/.test(failureGate)) { + throw new Error('--failure-gate must be a stable lowercase gate ID'); + } const report = { schemaVersion: 'acceptance-report/v1', runId: runtime.runId, @@ -202,7 +230,7 @@ async function buildLocalPartial(values) { } : null, gates: [ { - id: 'local_execution_complete', + id: failureGate, passed: false, detail: `stopped during ${values['failure-phase']}` }, @@ -221,8 +249,7 @@ async function buildLocalPartial(values) { secretSafe: true }; validate(report); - await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 }); - await writeFile(resolve(values.output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + await writeNewJSON(values.output, report); process.stdout.write('acceptance_report_local_partial=PASS\n'); } @@ -259,8 +286,7 @@ async function mergeProduction(values) { }; local.createdAt = new Date().toISOString(); validate(local); - await mkdir(dirname(resolve(values.output)), { recursive: true, mode: 0o700 }); - await writeFile(resolve(values.output), `${JSON.stringify(local, null, 2)}\n`, { mode: 0o600 }); + await writeNewJSON(values.output, local); process.stdout.write(`acceptance_report_production=PASS promotion_ready=${local.promotion.ready}\n`); } @@ -285,7 +311,7 @@ if (command === 'validate') { } report.promotion.trafficMode = 'live'; report.promotion.promotedAt = new Date().toISOString(); - await writeFile(resolve(output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + await replaceJSONAtomically(input, output, report); process.stdout.write('acceptance_report_promoted=PASS\n'); } else if (command === 'attach-monitor') { const input = values.input ?? ''; @@ -309,7 +335,7 @@ if (command === 'validate') { report.promotion.ready = false; report.promotion.trafficMode = 'validation'; } - await writeFile(resolve(output), `${JSON.stringify(report, null, 2)}\n`, { mode: 0o600 }); + await replaceJSONAtomically(input, output, report); process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`); } else { throw new Error('usage: report.mjs {validate|build-local|build-local-partial|merge-production|mark-promoted|attach-monitor} [options]'); diff --git a/scripts/acceptance/run-local-acceptance.sh b/scripts/acceptance/run-local-acceptance.sh index b129f49..32dafd2 100755 --- a/scripts/acceptance/run-local-acceptance.sh +++ b/scripts/acceptance/run-local-acceptance.sh @@ -7,13 +7,18 @@ private_root="$repository_root/.local-secrets/acceptance" state_root="$private_root/state" context=k3d-easyai-acceptance-local namespace=easyai +cluster_name=easyai-acceptance-local runtime_path_file="$state_root/current-runtime-path" snapshot="$state_root/snapshot.json" load_binary="$state_root/easyai-ai-gateway-acceptance-load" +identity_file="$state_root/control-plane-identity.json" active_load_pid= netem_active=false current_phase=initialization stable_profile= +failure_gate_id=local_execution_incomplete +# shellcheck source=scripts/acceptance/local-control-plane.sh +source "$script_dir/local-control-plane.sh" usage() { cat <<'EOF' @@ -51,12 +56,16 @@ cleanup() { fi if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then restore_profile_best_effort "${stable_profile:-P24}" + if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then + failure_gate_id=$(jq -r '.id' "$failure_gate_file" 2>/dev/null || printf '%s' "$failure_gate_id") + fi local partial_args=( "$script_dir/report.mjs" build-local-partial --runtime "$runtime" \ --snapshot "$snapshot" \ --reports "$report_root" \ --failure-phase "$current_phase" \ + --failure-gate "$failure_gate_id" \ --output "$report_root/acceptance-report.partial.json" ) if [[ -n $stable_profile ]]; then @@ -89,6 +98,7 @@ require_local_cluster() { release_sha=$(jq -r '.releaseSha' "$runtime") report_root="$repository_root/dist/acceptance/local/$run_id" install -d -m 0700 "$report_root" + failure_gate_file="$report_root/failure-gate.json" ( cd "$repository_root/apps/api" env -u AI_GATEWAY_TEST_DATABASE_URL go build -trimpath \ @@ -96,6 +106,27 @@ require_local_cluster() { ) } +record_failure_gate() { + local id=$1 detail=$2 temporary + [[ $id =~ ^[a-z0-9_]+$ ]] + failure_gate_id=$id + [[ -n ${failure_gate_file:-} ]] || return 0 + if [[ ! -e $failure_gate_file ]]; then + temporary="${failure_gate_file}.tmp.$$" + jq -n --arg id "$id" --arg detail "$detail" \ + '{schemaVersion:"acceptance-failure-gate/v1",id:$id,detail:$detail}' >"$temporary" + chmod 0600 "$temporary" + mv "$temporary" "$failure_gate_file" + fi +} + +fail_gate() { + local id=$1 detail=$2 + record_failure_gate "$id" "$detail" + echo "gate=$id $detail" >&2 + return 1 +} + database_query() { local sql=$1 primary primary=$(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \ @@ -115,6 +146,12 @@ load_environment() { run_load() { local profile=$1 report_path=$2 shift 2 + local stdout_path="$report_path.stdout" stdout_temporary="${report_path}.stdout.tmp.$$" + local load_pid load_status=0 unavailable_samples=0 infrastructure_failure=false + [[ ! -e $report_path && ! -e $stdout_path && ! -e $stdout_temporary ]] || + fail_gate acceptance_report_exists "refusing to overwrite an existing load report" + verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || + fail_gate local_control_plane_restarted "local K3s server identity changed before load" AI_GATEWAY_ACCEPTANCE_GATEWAYS='https://127.0.0.1:18443,https://127.0.0.1:19443' \ AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=gateway.easyai.local \ AI_GATEWAY_ACCEPTANCE_GATEWAY_CA_FILE="$state_root/ca.crt" \ @@ -125,8 +162,49 @@ run_load() { AI_GATEWAY_ACCEPTANCE_GEMINI_MODEL="$acceptance_gemini_model" \ AI_GATEWAY_ACCEPTANCE_VIDEO_MODEL="$acceptance_video_model" \ "$load_binary" -profile "$profile" -report "$report_path" "$@" \ - >"$report_path.stdout" - chmod 0600 "$report_path" "$report_path.stdout" + >"$stdout_temporary" & + load_pid=$! + active_load_pid=$load_pid + while kill -0 "$load_pid" >/dev/null 2>&1; do + if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then + record_failure_gate local_control_plane_restarted "local K3s server restarted during load" + infrastructure_failure=true + kill "$load_pid" >/dev/null 2>&1 || true + break + fi + if verify_local_apiserver_ready "$context"; then + unavailable_samples=0 + else + unavailable_samples=$((unavailable_samples + 1)) + if ((unavailable_samples >= 3)); then + record_failure_gate local_control_plane_unavailable "Kubernetes API readiness failed for three consecutive samples" + infrastructure_failure=true + kill "$load_pid" >/dev/null 2>&1 || true + break + fi + fi + sleep 2 + done + if wait "$load_pid"; then + load_status=0 + else + load_status=$? + fi + active_load_pid= + if ! verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file"; then + record_failure_gate local_control_plane_restarted "local K3s server identity changed at load completion" + infrastructure_failure=true + fi + [[ ! -e $stdout_path ]] || fail_gate acceptance_report_exists "load stdout report already exists" + mv "$stdout_temporary" "$stdout_path" + chmod 0600 "$stdout_path" + if [[ -f $report_path && ! -L $report_path ]]; then + chmod 0600 "$report_path" + fi + if [[ $infrastructure_failure == true ]]; then + return 1 + fi + ((load_status == 0)) || return "$load_status" jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \ "$report_path" >/dev/null } @@ -147,15 +225,26 @@ sample_resources() { verify_hard_gates() { local unhealthy ready sync_state connections max_connections queue duplicate_remote duplicate_billing + verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || + fail_gate local_control_plane_restarted "local K3s server identity changed" + verify_local_apiserver_ready "$context" || + fail_gate local_control_plane_unavailable "Kubernetes API readiness failed" + verify_local_node_capacity "$context" || + fail_gate local_node_capacity_mismatch "node allocatable resources drifted from the acceptance envelope" + verify_local_etcd_runtime_logs "$cluster_name" "$identity_file" || + fail_gate local_etcd_latency "etcd emitted a severe latency or timeout signal during the run" ready=$(kubectl --context "$context" get nodes -o json | jq '[.items[] | select(any(.status.conditions[]; .type=="Ready" and .status=="True"))] | length') - [[ $ready == 3 ]] + [[ $ready == 3 ]] || fail_gate local_nodes_not_ready "not all three local K3s nodes are Ready" [[ $(kubectl --context "$context" get nodes -o json | - jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] + jq '[.items[] | select(any(.status.conditions[]; .type=="MemoryPressure" and .status=="True"))] | length') == 0 ]] || + fail_gate local_node_memory_pressure "a local K3s node reports MemoryPressure" [[ $(kubectl --context "$context" -n "$namespace" get cluster easyai-postgres \ - -o 'jsonpath={.status.readyInstances}') == 2 ]] + -o 'jsonpath={.status.readyInstances}') == 2 ]] || + fail_gate local_postgres_not_ready "local PostgreSQL is not 2/2 Ready" sync_state=$(database_query "SELECT COALESCE(string_agg(sync_state,','),'') FROM pg_stat_replication;") - [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] + [[ ",$sync_state," == *,sync,* || ",$sync_state," == *,quorum,* ]] || + fail_gate local_postgres_not_synchronous "local PostgreSQL lost its synchronous replica" unhealthy=$(kubectl --context "$context" -n "$namespace" get pods -o json | jq '[.items[] | select(.metadata.labels["app.kubernetes.io/name"] == "easyai-api" or @@ -163,11 +252,11 @@ verify_hard_gates() { | .status.containerStatuses[]? | select(.restartCount > 0 or .lastState.terminated.reason == "OOMKilled" or .state.terminated.reason == "OOMKilled")] | length') - [[ $unhealthy == 0 ]] + [[ $unhealthy == 0 ]] || fail_gate gateway_pod_restarted "an API or Worker container restarted or was OOMKilled" while read -r _node _cpu _cpu_percent _memory memory_percent; do memory_percent=${memory_percent%\%} [[ $memory_percent =~ ^[0-9]+$ ]] - (( memory_percent < 80 )) + (( memory_percent < 80 )) || fail_gate local_node_memory_target "local node memory reached the 80 percent target" done < <(kubectl --context "$context" top nodes --no-headers) while read -r _pod _cpu memory; do case $memory in @@ -176,20 +265,23 @@ verify_hard_gates() { *Ki) memory=$(awk -v value="${memory%Ki}" 'BEGIN {printf "%.0f", value/1024}') ;; *) return 1 ;; esac - (( memory < 1536 )) + (( memory < 1536 )) || fail_gate gateway_pod_memory_limit "an API or Worker Pod reached 1.5 GiB RSS" done < <(kubectl --context "$context" -n "$namespace" top pods \ -l 'app.kubernetes.io/part-of=easyai-ai-gateway' --no-headers) connections=$(database_query "SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend';") max_connections=$(database_query "SELECT setting::int FROM pg_settings WHERE name='max_connections';") - (( connections < 150 && connections * 4 < max_connections * 3 )) + (( connections < 150 && connections * 4 < max_connections * 3 )) || + fail_gate postgres_connection_budget "PostgreSQL client sessions exceeded the acceptance budget" queue=$(database_query "SELECT count(*) FILTER (WHERE status='queued')||':'||count(*) FILTER (WHERE status='running') FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;") - [[ $queue == 0:0 ]] + [[ $queue == 0:0 ]] || fail_gate acceptance_queue_not_drained "acceptance queue did not return to zero" duplicate_remote=$(database_query "SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) d;") duplicate_billing=$(database_query "SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) d;") - [[ $duplicate_remote == 0 && $duplicate_billing == 0 ]] + [[ $duplicate_remote == 0 ]] || fail_gate duplicate_upstream_submission "duplicate remote task IDs were detected" + [[ $duplicate_billing == 0 ]] || fail_gate duplicate_billing "duplicate billing transactions were detected" kubectl --context "$context" -n "$namespace" exec deployment/easyai-acceptance-callback-collector -- \ wget -qO- http://127.0.0.1:8091/report | - jq -e '.duplicates == 0 and .invalid == 0' >/dev/null + jq -e '.duplicates == 0 and .invalid == 0' >/dev/null || + fail_gate callback_validation_failed "callback collector detected duplicate or invalid callbacks" } apply_profile() { @@ -218,7 +310,8 @@ apply_profile() { AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \ AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \ - AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null + AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \ + AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m done for deployment in easyai-api-ningbo easyai-api-hongkong easyai-capacity-controller; do @@ -254,7 +347,8 @@ restore_profile_best_effort() { AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$slots" \ AI_GATEWAY_DATABASE_MAX_CONNS="$pool" \ AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY="$slots" \ - AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" >/dev/null 2>&1 || true + AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY="$slots" \ + AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=1 >/dev/null 2>&1 || true done } @@ -514,23 +608,29 @@ full() { command=${1:-} shift || true -require_local_cluster -load_environment case $command in quick) [[ $# -eq 0 ]] || { usage >&2; exit 64; } - quick ;; - artifact-smoke) + artifact-smoke|full) [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; } - artifact_smoke "$2" - ;; - full) - [[ ${1:-} == --release-manifest && $# -eq 2 ]] || { usage >&2; exit 64; } - full "$2" ;; *) usage >&2 exit 64 ;; esac +"$script_dir/local-cluster.sh" new-run +require_local_cluster +load_environment +case $command in + quick) + quick + ;; + artifact-smoke) + artifact_smoke "$2" + ;; + full) + full "$2" + ;; +esac diff --git a/tests/acceptance/report-test.mjs b/tests/acceptance/report-test.mjs index d6e612f..c67dc38 100644 --- a/tests/acceptance/report-test.mjs +++ b/tests/acceptance/report-test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -73,6 +73,19 @@ assert.equal(report.stages.amd64Artifact.status, 'passed'); assert.equal(report.promotion.ready, false); assert.equal(readFileSync(output, 'utf8').includes('must-not-be-copied'), false); execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', output]); +const overwrite = spawnSync('node', [ + join(root, 'scripts/acceptance/report.mjs'), + 'build-local', + '--runtime', runtime, + '--snapshot', snapshot, + '--reports', reports, + '--artifact', artifact, + '--api-digest', digest, + '--output', output +], { encoding: 'utf8' }); +assert.notEqual(overwrite.status, 0); +assert.match(`${overwrite.stdout}${overwrite.stderr}`, /EEXIST|file already exists/); +assert.equal(JSON.parse(readFileSync(output, 'utf8')).runId, runID); execFileSync('node', [ join(root, 'scripts/acceptance/report.mjs'), @@ -82,11 +95,13 @@ execFileSync('node', [ '--reports', reports, '--certified-profile', 'P24', '--failure-phase', 'capacity_p28_round_2', + '--failure-gate', 'local_control_plane_restarted', '--output', partialOutput ]); const partialReport = JSON.parse(readFileSync(partialOutput, 'utf8')); assert.equal(partialReport.stages.localNative.status, 'failed'); assert.equal(partialReport.stages.localNative.failurePhase, 'capacity_p28_round_2'); +assert.equal(partialReport.gates[0].id, 'local_control_plane_restarted'); assert.equal(partialReport.stages.amd64Artifact.status, 'not-run'); assert.equal(partialReport.promotion.ready, false); assert.equal(readFileSync(partialOutput, 'utf8').includes('must-not-be-copied'), false);