fix(acceptance): 阻断本地控制面漂移污染验收
本地 K3s 节点此前在 Kubelet 中登记为宿主机资源,负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像,并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID,报告使用独占或原子写入,负载错误记录具体阶段;数据库鉴权不可用返回带 Retry-After 的 503,避免基础设施故障被误报为 401。\n\n验证:Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
This commit is contained in:
@@ -119,6 +119,12 @@ func run(ctx context.Context, opts options) error {
|
||||
return err
|
||||
}
|
||||
defer database.Close()
|
||||
if err := verifyLocalClusterMarker(ctx, database, opts.clusterID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := resetPreviousLocalRun(ctx, database, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verifyLocalDatabase(ctx, database, opts.clusterID); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -212,7 +218,15 @@ func run(ctx context.Context, opts options) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(outputPath, append(payload, '\n'), 0o600); err != nil {
|
||||
outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := outputFile.Write(append(payload, '\n')); err != nil {
|
||||
_ = outputFile.Close()
|
||||
return err
|
||||
}
|
||||
if err := outputFile.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(
|
||||
@@ -226,7 +240,60 @@ func run(ctx context.Context, opts options) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error {
|
||||
mode, err := database.GetGatewayTrafficMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mode.Mode == "live" {
|
||||
return nil
|
||||
}
|
||||
if mode.Mode != "validation" || strings.TrimSpace(mode.RunID) == "" {
|
||||
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
|
||||
}
|
||||
run, err := database.GetAcceptanceRun(ctx, mode.RunID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if run.Config["localClusterId"] != opts.clusterID {
|
||||
return errors.New("refusing to replace an acceptance run owned by another cluster")
|
||||
}
|
||||
var outstanding int
|
||||
if err := database.Pool().QueryRow(ctx, `
|
||||
SELECT count(*)
|
||||
FROM gateway_tasks
|
||||
WHERE acceptance_run_id = $1::uuid
|
||||
AND status IN ('queued','running')`, run.ID).Scan(&outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
if outstanding != 0 {
|
||||
return fmt.Errorf("refusing to replace local acceptance run with %d outstanding tasks", outstanding)
|
||||
}
|
||||
_, err = database.AbortAcceptanceRun(ctx, store.PromoteAcceptanceRunInput{
|
||||
RunID: mode.RunID,
|
||||
Revision: mode.Revision,
|
||||
ReleaseSHA: mode.ReleaseSHA,
|
||||
APIImageDigest: mode.APIImageDigest,
|
||||
WorkerImageDigest: mode.WorkerImageDigest,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func verifyLocalDatabase(ctx context.Context, database *store.Store, clusterID string) error {
|
||||
if err := verifyLocalClusterMarker(ctx, database, clusterID); err != nil {
|
||||
return err
|
||||
}
|
||||
mode, err := database.GetGatewayTrafficMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mode.Mode != "live" {
|
||||
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyLocalClusterMarker(ctx context.Context, database *store.Store, clusterID string) error {
|
||||
var marker string
|
||||
err := database.Pool().QueryRow(ctx, `
|
||||
SELECT COALESCE(value->>'clusterId', '')
|
||||
@@ -238,13 +305,6 @@ WHERE setting_key = $1`, acceptancesnapshot.LocalClusterSettingKey).Scan(&marker
|
||||
if marker != clusterID {
|
||||
return errors.New("refusing bootstrap: local cluster marker mismatch")
|
||||
}
|
||||
mode, err := database.GetGatewayTrafficMode(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mode.Mode != "live" {
|
||||
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -486,10 +546,8 @@ func privateOutputPath(path string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info, err := os.Lstat(absolute); err == nil {
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", errors.New("runtime output must be a regular non-symlink file")
|
||||
}
|
||||
if _, err := os.Lstat(absolute); err == nil {
|
||||
return "", errors.New("runtime output already exists")
|
||||
} else if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -59,15 +59,16 @@ type options struct {
|
||||
}
|
||||
|
||||
type report struct {
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
RunID string `json:"runId"`
|
||||
Profile string `json:"profile"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt"`
|
||||
Passed bool `json:"passed"`
|
||||
Phases []phaseReport `json:"phases"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
SecretSafe bool `json:"secretSafe"`
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
RunID string `json:"runId"`
|
||||
Profile string `json:"profile"`
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
FinishedAt time.Time `json:"finishedAt"`
|
||||
Passed bool `json:"passed"`
|
||||
Phases []phaseReport `json:"phases"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
FailureOperation string `json:"failureOperation,omitempty"`
|
||||
SecretSafe bool `json:"secretSafe"`
|
||||
}
|
||||
|
||||
type phaseReport struct {
|
||||
@@ -101,6 +102,25 @@ type httpStatusError struct {
|
||||
Body string
|
||||
}
|
||||
|
||||
type operationError struct {
|
||||
Operation string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *operationError) Error() string { return e.Operation + ": " + e.Err.Error() }
|
||||
func (e *operationError) Unwrap() error { return e.Err }
|
||||
|
||||
func withOperation(operation string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var existing *operationError
|
||||
if errors.As(err, &existing) {
|
||||
return err
|
||||
}
|
||||
return &operationError{Operation: operation, Err: err}
|
||||
}
|
||||
|
||||
func (e *httpStatusError) Error() string {
|
||||
return fmt.Sprintf("HTTP %d: %s", e.Status, e.Body)
|
||||
}
|
||||
@@ -137,11 +157,15 @@ func main() {
|
||||
result.Passed = runErr == nil
|
||||
if runErr != nil {
|
||||
result.Failure = redactError(runErr.Error(), opts)
|
||||
var operationErr *operationError
|
||||
if errors.As(runErr, &operationErr) {
|
||||
result.FailureOperation = operationErr.Operation
|
||||
}
|
||||
}
|
||||
payload, _ := json.MarshalIndent(result, "", " ")
|
||||
fmt.Println(string(payload))
|
||||
if opts.reportPath != "" {
|
||||
if err := os.WriteFile(opts.reportPath, append(payload, '\n'), 0o600); err != nil {
|
||||
if err := writeReportExclusive(opts.reportPath, payload); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "write acceptance report failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -151,6 +175,18 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
func writeReportExclusive(path string, payload []byte) error {
|
||||
reportFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := reportFile.Write(append(payload, '\n')); err != nil {
|
||||
_ = reportFile.Close()
|
||||
return err
|
||||
}
|
||||
return reportFile.Close()
|
||||
}
|
||||
|
||||
func parseOptions() (options, error) {
|
||||
var profile string
|
||||
var reportPath string
|
||||
@@ -347,14 +383,15 @@ func runGemini(
|
||||
response, err = client.Do(req)
|
||||
if err == nil {
|
||||
if response.StatusCode != http.StatusOK {
|
||||
err = responseStatusError(response)
|
||||
err = withOperation("gemini_submit", responseStatusError(response))
|
||||
} else {
|
||||
var size int64
|
||||
var outputHash string
|
||||
size, outputHash, err = streamGeminiImageHash(response.Body)
|
||||
err = withOperation("gemini_response", err)
|
||||
_ = response.Body.Close()
|
||||
if err == nil && expectedOutputBytes > 0 && (size != int64(expectedOutputBytes) || outputHash != expectedHash) {
|
||||
err = fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash)
|
||||
err = withOperation("gemini_response", fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash))
|
||||
}
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
@@ -367,6 +404,7 @@ func runGemini(
|
||||
}
|
||||
}
|
||||
}
|
||||
err = withOperation("gemini_submit", err)
|
||||
latency := time.Since(requestStarted)
|
||||
mu.Lock()
|
||||
latencies[index] = latency
|
||||
@@ -503,7 +541,7 @@ func runVideo(
|
||||
response, err = client.Do(req)
|
||||
if err == nil {
|
||||
if response.StatusCode != http.StatusAccepted {
|
||||
err = responseStatusError(response)
|
||||
err = withOperation("video_submit", responseStatusError(response))
|
||||
} else {
|
||||
var accepted struct {
|
||||
TaskID string `json:"taskId"`
|
||||
@@ -517,6 +555,7 @@ func runVideo(
|
||||
}
|
||||
}
|
||||
}
|
||||
err = withOperation("video_submit", err)
|
||||
mu.Lock()
|
||||
latencies[index] = time.Since(requestStarted)
|
||||
if !realUpstream && logicalIndex%4 == 0 {
|
||||
@@ -742,7 +781,7 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI
|
||||
endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/ai/result/" + url.PathEscape(taskID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
return withOperation("video_poll", err)
|
||||
}
|
||||
opts.setHeaders(req, index, realUpstream)
|
||||
response, err := client.Do(req)
|
||||
@@ -758,20 +797,20 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI
|
||||
case "succeeded", "success":
|
||||
mediaURL := findMediaURL(payload)
|
||||
if mediaURL == "" {
|
||||
return fmt.Errorf("video task %s succeeded without a media URL", taskID)
|
||||
return withOperation("video_poll", fmt.Errorf("video task %s succeeded without a media URL", taskID))
|
||||
}
|
||||
return validateVideoAsset(ctx, client, opts, mediaURL, index)
|
||||
return withOperation("video_download", validateVideoAsset(ctx, client, opts, mediaURL, index))
|
||||
case "failed", "cancelled", "canceled":
|
||||
return fmt.Errorf("video task %s finished with status %s", taskID, status)
|
||||
return withOperation("video_poll", fmt.Errorf("video task %s finished with status %s", taskID, status))
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
return withOperation("video_poll", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return withOperation("video_poll", ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -212,6 +213,35 @@ func TestAcceptanceReportErrorRedactsSecretsAndURLs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceReportCannotOverwriteExistingRunArtifact(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "load-report.json")
|
||||
if err := writeReportExclusive(path, []byte(`{"runId":"first"}`)); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
if err := writeReportExclusive(path, []byte(`{"runId":"second"}`)); err == nil {
|
||||
t.Fatal("second report unexpectedly overwrote the first report")
|
||||
}
|
||||
payload, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read report: %v", err)
|
||||
}
|
||||
if string(payload) != "{\"runId\":\"first\"}\n" {
|
||||
t.Fatalf("report changed after rejected overwrite: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceFailurePreservesOperationAndHTTPStatus(t *testing.T) {
|
||||
err := withOperation("video_poll", &httpStatusError{Status: http.StatusUnauthorized, Body: "unauthorized"})
|
||||
var operationErr *operationError
|
||||
if !errors.As(err, &operationErr) || operationErr.Operation != "video_poll" {
|
||||
t.Fatalf("operation error=%#v", operationErr)
|
||||
}
|
||||
var statusErr *httpStatusError
|
||||
if !errors.As(err, &statusErr) || statusErr.Status != http.StatusUnauthorized {
|
||||
t.Fatalf("HTTP status error=%#v", statusErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcceptanceGatewayTLSNameSetsHostHeader(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "https://127.0.0.1/api/v1/healthz", nil)
|
||||
opts := options{
|
||||
|
||||
@@ -5443,6 +5443,12 @@
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.EasyAIGeneratedResponse"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6323,6 +6329,12 @@
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.VolcesErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.VolcesErrorEnvelope"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7405,6 +7417,12 @@
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.GeminiErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.GeminiErrorEnvelope"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7478,6 +7496,12 @@
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.GeminiErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.GeminiErrorEnvelope"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7578,6 +7602,12 @@
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"504": {
|
||||
"description": "Gateway Timeout",
|
||||
"schema": {
|
||||
@@ -8446,6 +8476,12 @@
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"504": {
|
||||
"description": "Gateway Timeout",
|
||||
"schema": {
|
||||
@@ -8551,6 +8587,12 @@
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"504": {
|
||||
"description": "Gateway Timeout",
|
||||
"schema": {
|
||||
@@ -8999,6 +9041,12 @@
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"504": {
|
||||
"description": "Gateway Timeout",
|
||||
"schema": {
|
||||
@@ -9324,6 +9372,12 @@
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"503": {
|
||||
"description": "Service Unavailable",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/httpapi.ErrorEnvelope"
|
||||
}
|
||||
},
|
||||
"504": {
|
||||
"description": "Gateway Timeout",
|
||||
"schema": {
|
||||
|
||||
@@ -7680,6 +7680,10 @@ paths:
|
||||
description: Gone
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.EasyAIGeneratedResponse'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: 查询 server-main EasyAIClient 兼容任务结果
|
||||
@@ -8248,6 +8252,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.VolcesErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.VolcesErrorEnvelope'
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: 创建火山内容生成任务
|
||||
@@ -8945,6 +8953,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.GeminiErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.GeminiErrorEnvelope'
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Gemini generateContent 兼容接口
|
||||
@@ -8993,6 +9005,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.GeminiErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.GeminiErrorEnvelope'
|
||||
security:
|
||||
- BearerAuth: []
|
||||
summary: Gemini generateContent 兼容接口
|
||||
@@ -9058,6 +9074,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"504":
|
||||
description: Gateway Timeout
|
||||
schema:
|
||||
@@ -9623,6 +9643,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"504":
|
||||
description: Gateway Timeout
|
||||
schema:
|
||||
@@ -9692,6 +9716,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"504":
|
||||
description: Gateway Timeout
|
||||
schema:
|
||||
@@ -9980,6 +10008,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"504":
|
||||
description: Gateway Timeout
|
||||
schema:
|
||||
@@ -10192,6 +10224,10 @@ paths:
|
||||
description: Bad Gateway
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"503":
|
||||
description: Service Unavailable
|
||||
schema:
|
||||
$ref: '#/definitions/httpapi.ErrorEnvelope'
|
||||
"504":
|
||||
description: Gateway Timeout
|
||||
schema:
|
||||
|
||||
@@ -69,9 +69,10 @@ const userContextKey contextKey = "easyai-auth-user"
|
||||
var ErrUnauthorized = errors.New("unauthorized")
|
||||
|
||||
type RequestAuthError struct {
|
||||
Status int
|
||||
Code string
|
||||
Message string
|
||||
Status int
|
||||
Code string
|
||||
Message string
|
||||
RetryAfterSeconds int
|
||||
}
|
||||
|
||||
func (e *RequestAuthError) Error() string { return e.Code }
|
||||
@@ -80,6 +81,12 @@ func NewRequestAuthError(status int, code, message string) error {
|
||||
return &RequestAuthError{Status: status, Code: code, Message: message}
|
||||
}
|
||||
|
||||
func NewRetryableRequestAuthError(status int, code, message string, retryAfterSeconds int) error {
|
||||
return &RequestAuthError{
|
||||
Status: status, Code: code, Message: message, RetryAfterSeconds: retryAfterSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
type Authenticator struct {
|
||||
JWTSecret string
|
||||
ServerMainBaseURL string
|
||||
@@ -127,6 +134,9 @@ func (a *Authenticator) Require(permission Permission, next http.Handler) http.H
|
||||
}
|
||||
var requestError *RequestAuthError
|
||||
if errors.As(err, &requestError) {
|
||||
if requestError.RetryAfterSeconds > 0 {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(requestError.RetryAfterSeconds))
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.WriteHeader(requestError.Status)
|
||||
@@ -318,7 +328,12 @@ func (a *Authenticator) verifyAPIKey(ctx context.Context, apiKey string) (*User,
|
||||
return user, nil
|
||||
}
|
||||
if !errors.Is(err, ErrUnauthorized) {
|
||||
return nil, err
|
||||
return nil, NewRetryableRequestAuthError(
|
||||
http.StatusServiceUnavailable,
|
||||
"AUTH_STORE_UNAVAILABLE",
|
||||
"authentication service temporarily unavailable",
|
||||
2,
|
||||
)
|
||||
}
|
||||
if strings.HasPrefix(apiKey, "sk-gw-") {
|
||||
return nil, ErrUnauthorized
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAPIKeyStoreFailureReturnsRetryableServiceUnavailable(t *testing.T) {
|
||||
authenticator := New("test-secret", "", "")
|
||||
authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*User, error) {
|
||||
return nil, errors.New("database unavailable")
|
||||
}
|
||||
handler := authenticator.Require(PermissionBasic, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
||||
t.Fatal("unavailable authentication store must not call the protected handler")
|
||||
}))
|
||||
request := httptest.NewRequest(http.MethodGet, "/protected", nil)
|
||||
request.Header.Set("Authorization", "Bearer sk-gw-local")
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if recorder.Header().Get("Retry-After") != "2" {
|
||||
t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After"))
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
@@ -19,6 +20,9 @@ func (s *Server) requireProtocolUser(protocol string, next http.Handler) http.Ha
|
||||
var requestErr *auth.RequestAuthError
|
||||
if errors.As(err, &requestErr) {
|
||||
status, code, message = requestErr.Status, requestErr.Code, requestErr.Message
|
||||
if requestErr.RetryAfterSeconds > 0 {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(requestErr.RetryAfterSeconds))
|
||||
}
|
||||
}
|
||||
writeProtocolError(w, protocol, status, message, nil, code)
|
||||
return
|
||||
|
||||
@@ -3,6 +3,7 @@ package httpapi
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -13,6 +14,39 @@ import (
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
func TestProtocolAPIKeyStoreFailureIs503InsteadOf401(t *testing.T) {
|
||||
authenticator := auth.New("test-secret", "", "")
|
||||
authenticator.LocalAPIKeyVerifier = func(context.Context, string) (*auth.User, error) {
|
||||
return nil, errors.New("database unavailable")
|
||||
}
|
||||
server := &Server{auth: authenticator}
|
||||
handler := server.requireProtocolUser(clients.ProtocolGeminiGenerateContent, http.HandlerFunc(
|
||||
func(http.ResponseWriter, *http.Request) {
|
||||
t.Fatal("unavailable authentication store must not call the protocol handler")
|
||||
},
|
||||
))
|
||||
request := httptest.NewRequest(http.MethodPost, "/v1beta/models/test:generateContent", nil)
|
||||
request.Header.Set("Authorization", "Bearer sk-gw-local")
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
handler.ServeHTTP(recorder, request)
|
||||
|
||||
if recorder.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d, want 503; body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if recorder.Header().Get("Retry-After") != "2" {
|
||||
t.Fatalf("Retry-After=%q, want 2", recorder.Header().Get("Retry-After"))
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
errorBody := requireObject(t, body["error"])
|
||||
if errorBody["code"] != float64(http.StatusServiceUnavailable) || errorBody["status"] != "UNAVAILABLE" {
|
||||
t.Fatalf("unexpected Gemini unavailable response: %+v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolErrorsUseOfficialShapesWithoutGatewayExtensions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -106,6 +106,7 @@ func geminiGenerateContentModelFromPath(prefix string, requestPath string) (stri
|
||||
// @Failure 401 {object} GeminiErrorEnvelope
|
||||
// @Failure 403 {object} GeminiErrorEnvelope
|
||||
// @Failure 502 {object} GeminiErrorEnvelope
|
||||
// @Failure 503 {object} GeminiErrorEnvelope
|
||||
// @Router /api/v1/models/{model}:generateContent [post]
|
||||
// @Router /api/v1/models/{model}:streamGenerateContent [post]
|
||||
func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -1420,6 +1420,7 @@ func openAIImagesDoc() {}
|
||||
// @Failure 429 {object} ErrorEnvelope
|
||||
// @Failure 504 {object} ErrorEnvelope
|
||||
// @Failure 502 {object} ErrorEnvelope
|
||||
// @Failure 503 {object} ErrorEnvelope
|
||||
// @Router /api/v1/videos/generations [post]
|
||||
// @Router /api/v1/song/generations [post]
|
||||
// @Router /api/v1/music/generations [post]
|
||||
|
||||
@@ -28,6 +28,7 @@ const volcesContentsCompatibilityMarker = "volces_contents_generations_v3"
|
||||
// @Failure 401 {object} VolcesErrorEnvelope
|
||||
// @Failure 429 {object} VolcesErrorEnvelope
|
||||
// @Failure 502 {object} VolcesErrorEnvelope
|
||||
// @Failure 503 {object} VolcesErrorEnvelope
|
||||
// @Router /api/v1/contents/generations/tasks [post]
|
||||
func (s *Server) createVolcesContentsGenerationTask(w http.ResponseWriter, r *http.Request) {
|
||||
user, ok := auth.UserFromContext(r.Context())
|
||||
@@ -186,6 +187,7 @@ func (s *Server) createLegacyVolcesVideoGeneration(w http.ResponseWriter, r *htt
|
||||
// @Success 200 {object} EasyAIGeneratedResponse
|
||||
// @Failure 404 {object} EasyAIGeneratedResponse
|
||||
// @Failure 410 {object} EasyAIGeneratedResponse
|
||||
// @Failure 503 {object} ErrorEnvelope
|
||||
// @Router /api/v1/ai/result/{taskID} [get]
|
||||
func (s *Server) getEasyAITaskResult(w http.ResponseWriter, r *http.Request) {
|
||||
user, ok := auth.UserFromContext(r.Context())
|
||||
|
||||
Reference in New Issue
Block a user