fix(acceptance): 隔离控制面抖动与租约瞬态故障
线上 P24 验收暴露出高频 kubectl exec 放大 K3s API 压力、门禁查询挤占关键连接池，以及 PostgreSQL 锁超时被误判为租约所有权丢失。 本次合并验收身份查询、在租约有效期内重试瞬态续期错误、修复人工审核残留 attempt，并增加滚动后 etcd 稳定窗口、节点直连指标和双站独立报告。 验证：Go 全量测试、go vet、聚焦 race、gofmt、迁移安全检查、bash -n、ShellCheck、manual release test。
This commit is contained in:
@@ -1,21 +1,43 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
|
||||
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
|
||||
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
|
||||
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
|
||||
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
|
||||
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
|
||||
taskTrafficAuthorizationCacheTTL = 5 * time.Second
|
||||
)
|
||||
|
||||
type taskTrafficAuthorizationCacheKey struct {
|
||||
runID string
|
||||
apiKeyID string
|
||||
userID string
|
||||
tokenHash [sha256.Size]byte
|
||||
}
|
||||
|
||||
type taskTrafficAuthorizationCacheEntry struct {
|
||||
runID string
|
||||
expiresAt time.Time
|
||||
}
|
||||
|
||||
type taskTrafficAuthorizationCall struct {
|
||||
done chan struct{}
|
||||
runID string
|
||||
err error
|
||||
}
|
||||
|
||||
type taskTrafficAdmission struct {
|
||||
RunMode string
|
||||
AcceptanceRunID string
|
||||
@@ -44,12 +66,7 @@ func (e *taskTrafficError) ErrorCode() string {
|
||||
}
|
||||
|
||||
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
|
||||
runID, err := s.acceptanceStore().AuthorizeAcceptanceTask(
|
||||
r.Context(),
|
||||
r.Header.Get(acceptanceRunHeader),
|
||||
r.Header.Get(acceptanceTokenHeader),
|
||||
user,
|
||||
)
|
||||
runID, err := s.authorizeAcceptanceTask(r.Context(), r.Header.Get(acceptanceRunHeader), r.Header.Get(acceptanceTokenHeader), user)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrProductionTrafficPaused):
|
||||
@@ -84,6 +101,78 @@ func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTra
|
||||
return taskTrafficAdmission{RunMode: "production"}, nil
|
||||
}
|
||||
|
||||
// authorizeAcceptanceTask collapses concurrent validation requests for the same
|
||||
// participant into one critical-pool lookup and briefly caches only successful
|
||||
// validation grants. Live requests and incomplete acceptance credentials always
|
||||
// consult PostgreSQL so a validation transition remains fail-closed.
|
||||
func (s *Server) authorizeAcceptanceTask(ctx context.Context, runID string, token string, user *auth.User) (string, error) {
|
||||
runID = strings.TrimSpace(runID)
|
||||
token = strings.TrimSpace(token)
|
||||
if runID == "" || token == "" || user == nil ||
|
||||
strings.TrimSpace(user.APIKeyID) == "" || strings.TrimSpace(user.ID) == "" {
|
||||
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
|
||||
}
|
||||
key := taskTrafficAuthorizationCacheKey{
|
||||
runID: runID, apiKeyID: strings.TrimSpace(user.APIKeyID), userID: strings.TrimSpace(user.ID),
|
||||
tokenHash: sha256.Sum256([]byte(token)),
|
||||
}
|
||||
return s.cachedTaskTrafficAuthorization(ctx, key, taskTrafficAuthorizationCacheTTL, func() (string, error) {
|
||||
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) cachedTaskTrafficAuthorization(
|
||||
ctx context.Context,
|
||||
key taskTrafficAuthorizationCacheKey,
|
||||
ttl time.Duration,
|
||||
load func() (string, error),
|
||||
) (string, error) {
|
||||
now := time.Now()
|
||||
s.taskTrafficCacheMu.Lock()
|
||||
if entry, ok := s.taskTrafficCache[key]; ok {
|
||||
if now.Before(entry.expiresAt) {
|
||||
s.taskTrafficCacheMu.Unlock()
|
||||
return entry.runID, nil
|
||||
}
|
||||
delete(s.taskTrafficCache, key)
|
||||
}
|
||||
if call, ok := s.taskTrafficInflight[key]; ok {
|
||||
s.taskTrafficCacheMu.Unlock()
|
||||
select {
|
||||
case <-call.done:
|
||||
return call.runID, call.err
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
}
|
||||
}
|
||||
if s.taskTrafficInflight == nil {
|
||||
s.taskTrafficInflight = make(map[taskTrafficAuthorizationCacheKey]*taskTrafficAuthorizationCall)
|
||||
}
|
||||
call := &taskTrafficAuthorizationCall{done: make(chan struct{})}
|
||||
s.taskTrafficInflight[key] = call
|
||||
s.taskTrafficCacheMu.Unlock()
|
||||
|
||||
call.runID, call.err = load()
|
||||
|
||||
s.taskTrafficCacheMu.Lock()
|
||||
delete(s.taskTrafficInflight, key)
|
||||
if call.err == nil && call.runID != "" {
|
||||
if s.taskTrafficCache == nil {
|
||||
s.taskTrafficCache = make(map[taskTrafficAuthorizationCacheKey]taskTrafficAuthorizationCacheEntry)
|
||||
}
|
||||
callTTL := ttl
|
||||
if callTTL <= 0 {
|
||||
callTTL = taskTrafficAuthorizationCacheTTL
|
||||
}
|
||||
s.taskTrafficCache[key] = taskTrafficAuthorizationCacheEntry{
|
||||
runID: call.runID, expiresAt: time.Now().Add(callTTL),
|
||||
}
|
||||
}
|
||||
close(call.done)
|
||||
s.taskTrafficCacheMu.Unlock()
|
||||
return call.runID, call.err
|
||||
}
|
||||
|
||||
func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) {
|
||||
if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" {
|
||||
return admission.RunMode, nil
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
||||
)
|
||||
@@ -25,3 +29,75 @@ func TestAdmittedTaskRunModeReservesAcceptanceAndProductionSimulation(t *testing
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedTaskTrafficAuthorizationCollapsesConcurrentLoads(t *testing.T) {
|
||||
server := &Server{}
|
||||
key := taskTrafficAuthorizationCacheKey{runID: "run", apiKeyID: "key", userID: "user"}
|
||||
var loads atomic.Int32
|
||||
start := make(chan struct{})
|
||||
const callers = 64
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, callers)
|
||||
for range callers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
runID, err := server.cachedTaskTrafficAuthorization(
|
||||
context.Background(), key, time.Minute,
|
||||
func() (string, error) {
|
||||
loads.Add(1)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
return "run", nil
|
||||
},
|
||||
)
|
||||
if err != nil || runID != "run" {
|
||||
errCh <- errors.New("unexpected cached authorization result")
|
||||
}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loads.Load(); got != 1 {
|
||||
t.Fatalf("authorization loads=%d, want 1", got)
|
||||
}
|
||||
|
||||
if _, err := server.cachedTaskTrafficAuthorization(
|
||||
context.Background(), key, time.Minute,
|
||||
func() (string, error) {
|
||||
loads.Add(1)
|
||||
return "run", nil
|
||||
},
|
||||
); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := loads.Load(); got != 1 {
|
||||
t.Fatalf("cached authorization loads=%d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachedTaskTrafficAuthorizationDoesNotCacheFailures(t *testing.T) {
|
||||
server := &Server{}
|
||||
key := taskTrafficAuthorizationCacheKey{runID: "run", apiKeyID: "key", userID: "user"}
|
||||
wantErr := errors.New("database unavailable")
|
||||
loads := 0
|
||||
for range 2 {
|
||||
_, err := server.cachedTaskTrafficAuthorization(
|
||||
context.Background(), key, time.Minute,
|
||||
func() (string, error) {
|
||||
loads++
|
||||
return "", wantErr
|
||||
},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("authorization error=%v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
if loads != 2 {
|
||||
t.Fatalf("failed authorization loads=%d, want 2", loads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,9 @@ type Server struct {
|
||||
identityTestCookieSecure bool
|
||||
identityTestBrowserEnabled bool
|
||||
billingMetrics *ssfreceiver.Metrics
|
||||
taskTrafficCacheMu sync.Mutex
|
||||
taskTrafficCache map[taskTrafficAuthorizationCacheKey]taskTrafficAuthorizationCacheEntry
|
||||
taskTrafficInflight map[taskTrafficAuthorizationCacheKey]*taskTrafficAuthorizationCall
|
||||
}
|
||||
|
||||
type oidcPublicClient interface {
|
||||
|
||||
Reference in New Issue
Block a user