BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 阻断本地控制面漂移污染验收

Browse Source 
本地 K3s 节点此前在 Kubelet 中登记为宿主机资源，负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像，并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID，报告使用独占或原子写入，负载错误记录具体阶段；数据库鉴权不可用返回带 Retry-After 的 503，避免基础设施故障被误报为 401。\n\n验证：Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
This commit is contained in:
wangbo
2026-07-31 23:07:19 +08:00
parent 015ff8ea6c
commit a95184b5b6
20 changed files with 970 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/auth/auth.go
+19 -4
View File
@@ -69,9 +69,10 @@ const userContextKey contextKey = "easyai-auth-user"
var ErrUnauthorized = errors.New("unauthorized")
type RequestAuthError struct {
Status int
Code string
Message string
Status int
Code string
Message string
RetryAfterSeconds int
}
func (e *RequestAuthError) Error() string { return e.Code }
@@ -80,6 +81,12 @@ func NewRequestAuthError(status int, code, message string) error {
return &RequestAuthError{Status: status, Code: code, Message: message}
}
func NewRetryableRequestAuthError(status int, code, message string, retryAfterSeconds int) error {
return &RequestAuthError{
Status: status, Code: code, Message: message, RetryAfterSeconds: retryAfterSeconds,
}
}
type Authenticator struct {
JWTSecret string
ServerMainBaseURL string
@@ -127,6 +134,9 @@ func (a *Authenticator) Require(permission Permission, next http.Handler) http.H
}
var requestError *RequestAuthError
if errors.As(err, &requestError) {
if requestError.RetryAfterSeconds > 0 {
w.Header().Set("Retry-After", strconv.Itoa(requestError.RetryAfterSeconds))
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
w.WriteHeader(requestError.Status)
@@ -318,7 +328,12 @@ func (a *Authenticator) verifyAPIKey(ctx context.Context, apiKey string) (*User,
return user, nil
}
if !errors.Is(err, ErrUnauthorized) {
return nil, err
return nil, NewRetryableRequestAuthError(
http.StatusServiceUnavailable,
"AUTH_STORE_UNAVAILABLE",
"authentication service temporarily unavailable",
2,
)
}
if strings.HasPrefix(apiKey, "sk-gw-") {
return nil, ErrUnauthorized
apps/api/internal/auth/auth_store_unavailable_test.go
+31
View File
@@ -0,0 +1,31 @@
package auth
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
)
func TestAPIKeyStoreFailureReturnsRetryableServiceUnavailable(t *testing.T) {
authenticator := New("test-secret", "", "")
authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*User, error) {
return nil, errors.New("database unavailable")
}
handler := authenticator.Require(PermissionBasic, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
t.Fatal("unavailable authentication store must not call the protected handler")
}))
request := httptest.NewRequest(http.MethodGet, "/protected", nil)
request.Header.Set("Authorization", "Bearer sk-gw-local")
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Retry-After") != "2" {
t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After"))
}
}
apps/api/internal/httpapi/compat_protocol.go
+4
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"net/http"
"strconv"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
@@ -19,6 +20,9 @@ func (s *Server) requireProtocolUser(protocol string, next http.Handler) http.Ha
var requestErr *auth.RequestAuthError
if errors.As(err, &requestErr) {
status, code, message = requestErr.Status, requestErr.Code, requestErr.Message
if requestErr.RetryAfterSeconds > 0 {
w.Header().Set("Retry-After", strconv.Itoa(requestErr.RetryAfterSeconds))
}
}
writeProtocolError(w, protocol, status, message, nil, code)
return
apps/api/internal/httpapi/compat_protocol_test.go
+34
View File
@@ -3,6 +3,7 @@ package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
@@ -13,6 +14,39 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
authenticator := auth.New("test-secret", "", "")
authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*auth.User, error) {
return nil, errors.New("database unavailable")
}
server := &Server{auth: authenticator}
handler := server.requireProtocolUser(clients.ProtocolGeminiGenerateContent, http.HandlerFunc(
func(http.ResponseWriter, *http.Request) {
t.Fatal("unavailable authentication store must not call the protocol handler")
},
))
request := httptest.NewRequest(http.MethodPost, "/v1beta/models/test:generateContent", nil)
request.Header.Set("Authorization", "Bearer sk-gw-local")
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String())
}
if recorder.Header().Get("Retry-After") != "2" {
t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After"))
}
var body map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
t.Fatalf("decode response: %v", err)
}
errorBody := requireObject(t, body["error"])
if errorBody["code"] != float64(http.StatusServiceUnavailable) || errorBody["status"] != "UNAVAILABLE" {
t.Fatalf("unexpected Gemini unavailable response: %+v", body)
}
}
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
tests := []struct {
name string
apps/api/internal/httpapi/gemini_compat.go
+1
View File
@@ -106,6 +106,7 @@ func geminiGenerateContentModelFromPath(prefix string, requestPath string) (stri
// @Failure 401 {object} GeminiErrorEnvelope
// @Failure 403 {object} GeminiErrorEnvelope
// @Failure 502 {object} GeminiErrorEnvelope
// @Failure 503 {object} GeminiErrorEnvelope
// @Router /api/v1/models/{model}:generateContent [post]
// @Router /api/v1/models/{model}:streamGenerateContent [post]
func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
apps/api/internal/httpapi/handlers.go
+1
View File
@@ -1420,6 +1420,7 @@ func openAIImagesDoc() {}
// @Failure 429 {object} ErrorEnvelope
// @Failure 504 {object} ErrorEnvelope
// @Failure 502 {object} ErrorEnvelope
// @Failure 503 {object} ErrorEnvelope
// @Router /api/v1/videos/generations [post]
// @Router /api/v1/song/generations [post]
// @Router /api/v1/music/generations [post]
apps/api/internal/httpapi/volces_compat_handlers.go
+2
View File
@@ -28,6 +28,7 @@ const volcesContentsCompatibilityMarker = "volces_contents_generations_v3"
// @Failure 401 {object} VolcesErrorEnvelope
// @Failure 429 {object} VolcesErrorEnvelope
// @Failure 502 {object} VolcesErrorEnvelope
// @Failure 503 {object} VolcesErrorEnvelope
// @Router /api/v1/contents/generations/tasks [post]
func (s *Server) createVolcesContentsGenerationTask(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())
@@ -186,6 +187,7 @@ func (s *Server) createLegacyVolcesVideoGeneration(w http.ResponseWriter, r *htt
// @Success 200 {object} EasyAIGeneratedResponse
// @Failure 404 {object} EasyAIGeneratedResponse
// @Failure 410 {object} EasyAIGeneratedResponse
// @Failure 503 {object} ErrorEnvelope
// @Router /api/v1/ai/result/{taskID} [get]
func (s *Server) getEasyAITaskResult(w http.ResponseWriter, r *http.Request) {
user, ok := auth.UserFromContext(r.Context())