BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(acceptance): 隔离控制面抖动与租约瞬态故障

Browse Source 
线上 P24 验收暴露出高频 kubectl exec 放大 K3s API 压力、门禁查询挤占关键连接池，以及 PostgreSQL 锁超时被误判为租约所有权丢失。

本次合并验收身份查询、在租约有效期内重试瞬态续期错误、修复人工审核残留 attempt，并增加滚动后 etcd 稳定窗口、节点直连指标和双站独立报告。

验证：Go 全量测试、go vet、聚焦 race、gofmt、迁移安全检查、bash -n、ShellCheck、manual release test。
This commit is contained in:
wangbo
2026-08-01 01:51:44 +08:00
parent be6ce7f78a
commit 132cda35d8
10 changed files with 495 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/httpapi/acceptance_handlers.go
+98 -9
View File
@@ -1,21 +1,43 @@
package httpapi
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
taskTrafficAuthorizationCacheTTL = 5 * time.Second
)
type taskTrafficAuthorizationCacheKey struct {
runID string
apiKeyID string
userID string
tokenHash [sha256.Size]byte
}
type taskTrafficAuthorizationCacheEntry struct {
runID string
expiresAt time.Time
}
type taskTrafficAuthorizationCall struct {
done chan struct{}
runID string
err error
}
type taskTrafficAdmission struct {
RunMode string
AcceptanceRunID string
@@ -44,12 +66,7 @@ func (e *taskTrafficError) ErrorCode() string {
}
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
runID, err := s.acceptanceStore().AuthorizeAcceptanceTask(
r.Context(),
r.Header.Get(acceptanceRunHeader),
r.Header.Get(acceptanceTokenHeader),
user,
)
runID, err := s.authorizeAcceptanceTask(r.Context(), r.Header.Get(acceptanceRunHeader), r.Header.Get(acceptanceTokenHeader), user)
if err != nil {
switch {
case errors.Is(err, store.ErrProductionTrafficPaused):
@@ -84,6 +101,78 @@ func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTra
return taskTrafficAdmission{RunMode: "production"}, nil
}
// authorizeAcceptanceTask collapses concurrent validation requests for the same
// participant into one critical-pool lookup and briefly caches only successful
// validation grants. Live requests and incomplete acceptance credentials always
// consult PostgreSQL so a validation transition remains fail-closed.
func (s *Server) authorizeAcceptanceTask(ctx context.Context, runID string, token string, user *auth.User) (string, error) {
runID = strings.TrimSpace(runID)
token = strings.TrimSpace(token)
if runID == "" || token == "" || user == nil ||
strings.TrimSpace(user.APIKeyID) == "" || strings.TrimSpace(user.ID) == "" {
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
}
key := taskTrafficAuthorizationCacheKey{
runID: runID, apiKeyID: strings.TrimSpace(user.APIKeyID), userID: strings.TrimSpace(user.ID),
tokenHash: sha256.Sum256([]byte(token)),
}
return s.cachedTaskTrafficAuthorization(ctx, key, taskTrafficAuthorizationCacheTTL, func() (string, error) {
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
})
}
func (s *Server) cachedTaskTrafficAuthorization(
ctx context.Context,
key taskTrafficAuthorizationCacheKey,
ttl time.Duration,
load func() (string, error),
) (string, error) {
now := time.Now()
s.taskTrafficCacheMu.Lock()
if entry, ok := s.taskTrafficCache[key]; ok {
if now.Before(entry.expiresAt) {
s.taskTrafficCacheMu.Unlock()
return entry.runID, nil
}
delete(s.taskTrafficCache, key)
}
if call, ok := s.taskTrafficInflight[key]; ok {
s.taskTrafficCacheMu.Unlock()
select {
case <-call.done:
return call.runID, call.err
case <-ctx.Done():
return "", ctx.Err()
}
}
if s.taskTrafficInflight == nil {
s.taskTrafficInflight = make(map[taskTrafficAuthorizationCacheKey]*taskTrafficAuthorizationCall)
}
call := &taskTrafficAuthorizationCall{done: make(chan struct{})}
s.taskTrafficInflight[key] = call
s.taskTrafficCacheMu.Unlock()
call.runID, call.err = load()
s.taskTrafficCacheMu.Lock()
delete(s.taskTrafficInflight, key)
if call.err == nil && call.runID != "" {
if s.taskTrafficCache == nil {
s.taskTrafficCache = make(map[taskTrafficAuthorizationCacheKey]taskTrafficAuthorizationCacheEntry)
}
callTTL := ttl
if callTTL <= 0 {
callTTL = taskTrafficAuthorizationCacheTTL
}
s.taskTrafficCache[key] = taskTrafficAuthorizationCacheEntry{
runID: call.runID, expiresAt: time.Now().Add(callTTL),
}
}
close(call.done)
s.taskTrafficCacheMu.Unlock()
return call.runID, call.err
}
func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) {
if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" {
return admission.RunMode, nil
apps/api/internal/httpapi/acceptance_handlers_test.go
+76
View File
@@ -1,8 +1,12 @@
package httpapi
import (
"context"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
)
@@ -25,3 +29,75 @@ func TestAdmittedTaskRunModeReservesAcceptanceAndProductionSimulation(t *testing
}
}
}
func TestCachedTaskTrafficAuthorizationCollapsesConcurrentLoads(t *testing.T) {
server := &Server{}
key := taskTrafficAuthorizationCacheKey{runID: "run", apiKeyID: "key", userID: "user"}
var loads atomic.Int32
start := make(chan struct{})
const callers = 64
var wg sync.WaitGroup
errCh := make(chan error, callers)
for range callers {
wg.Add(1)
go func() {
defer wg.Done()
<-start
runID, err := server.cachedTaskTrafficAuthorization(
context.Background(), key, time.Minute,
func() (string, error) {
loads.Add(1)
time.Sleep(10 * time.Millisecond)
return "run", nil
},
)
if err != nil || runID != "run" {
errCh <- errors.New("unexpected cached authorization result")
}
}()
}
close(start)
wg.Wait()
close(errCh)
for err := range errCh {
t.Fatal(err)
}
if got := loads.Load(); got != 1 {
t.Fatalf("authorization loads=%d, want 1", got)
}
if _, err := server.cachedTaskTrafficAuthorization(
context.Background(), key, time.Minute,
func() (string, error) {
loads.Add(1)
return "run", nil
},
); err != nil {
t.Fatal(err)
}
if got := loads.Load(); got != 1 {
t.Fatalf("cached authorization loads=%d, want 1", got)
}
}
func TestCachedTaskTrafficAuthorizationDoesNotCacheFailures(t *testing.T) {
server := &Server{}
key := taskTrafficAuthorizationCacheKey{runID: "run", apiKeyID: "key", userID: "user"}
wantErr := errors.New("database unavailable")
loads := 0
for range 2 {
_, err := server.cachedTaskTrafficAuthorization(
context.Background(), key, time.Minute,
func() (string, error) {
loads++
return "", wantErr
},
)
if !errors.Is(err, wantErr) {
t.Fatalf("authorization error=%v, want %v", err, wantErr)
}
}
if loads != 2 {
t.Fatalf("failed authorization loads=%d, want 2", loads)
}
}
apps/api/internal/httpapi/server.go
+3
View File
@@ -51,6 +51,9 @@ type Server struct {
identityTestCookieSecure bool
identityTestBrowserEnabled bool
billingMetrics *ssfreceiver.Metrics
taskTrafficCacheMu sync.Mutex
taskTrafficCache map[taskTrafficAuthorizationCacheKey]taskTrafficAuthorizationCacheEntry
taskTrafficInflight map[taskTrafficAuthorizationCacheKey]*taskTrafficAuthorizationCall
}
type oidcPublicClient interface {
apps/api/internal/runner/execution_lease.go
+19 -6
View File
@@ -11,24 +11,37 @@ import (
const (
taskExecutionLeaseTTL = 5 * time.Minute
taskExecutionRenewInterval = 30 * time.Second
leaseRenewalRetryInterval = 2 * time.Second
)
func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.CancelFunc, taskID string, executionToken string) {
ticker := time.NewTicker(taskExecutionRenewInterval)
defer ticker.Stop()
timer := time.NewTimer(taskExecutionRenewInterval)
defer timer.Stop()
leaseDeadline := time.Now().Add(taskExecutionLeaseTTL)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
case <-timer.C:
if err := s.coordinationStore.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil {
if errors.Is(err, store.ErrTaskExecutionFinished) {
return
}
s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost")
cancel()
return
if taskExecutionLeaseRenewalIsFatal(err, time.Now(), leaseDeadline) {
s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost", "error", err)
cancel()
return
}
s.logger.Warn("task execution lease renewal failed; retrying before lease expiry", "taskID", taskID, "error_category", "task_execution_lease_renewal_transient", "error", err)
timer.Reset(leaseRenewalRetryInterval)
continue
}
leaseDeadline = time.Now().Add(taskExecutionLeaseTTL)
timer.Reset(taskExecutionRenewInterval)
}
}
}
func taskExecutionLeaseRenewalIsFatal(err error, now time.Time, leaseDeadline time.Time) bool {
return errors.Is(err, store.ErrTaskExecutionLeaseLost) || !now.Before(leaseDeadline)
}
apps/api/internal/runner/execution_lease_test.go
+33
View File
@@ -0,0 +1,33 @@
package runner
import (
"errors"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestLeaseRenewalFatalityDistinguishesTransientDatabaseErrors(t *testing.T) {
now := time.Now()
deadline := now.Add(time.Minute)
transient := errors.New("lock timeout")
if taskExecutionLeaseRenewalIsFatal(transient, now, deadline) {
t.Fatal("transient task lease renewal error must be retried before expiry")
}
if concurrencyLeaseRenewalIsFatal(transient, now, deadline) {
t.Fatal("transient concurrency lease renewal error must be retried before expiry")
}
if !taskExecutionLeaseRenewalIsFatal(store.ErrTaskExecutionLeaseLost, now, deadline) {
t.Fatal("confirmed task lease ownership loss must be fatal")
}
if !concurrencyLeaseRenewalIsFatal(store.ErrConcurrencyLeaseLost, now, deadline) {
t.Fatal("confirmed concurrency lease ownership loss must be fatal")
}
if !taskExecutionLeaseRenewalIsFatal(transient, deadline, deadline) {
t.Fatal("task lease renewal error at expiry must be fatal")
}
if !concurrencyLeaseRenewalIsFatal(transient, deadline, deadline) {
t.Fatal("concurrency lease renewal error at expiry must be fatal")
}
}
apps/api/internal/runner/service.go
+32 -12
View File
@@ -2107,32 +2107,48 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
runCtx, cancelRun := context.WithCancel(ctx)
renewCtx, cancelRenew := context.WithCancel(ctx)
done := make(chan error, 1)
minimumTTL := 120 * time.Second
for _, lease := range leases {
if lease.TTL > 0 && lease.TTL < minimumTTL {
minimumTTL = lease.TTL
}
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
timer := time.NewTimer(interval)
defer timer.Stop()
leaseDeadline := time.Now().Add(minimumTTL)
for {
select {
case <-renewCtx.Done():
done <- nil
return
case <-ticker.C:
case <-timer.C:
if err := s.coordinationStore.RenewConcurrencyLeases(renewCtx, leases); err != nil {
if renewCtx.Err() != nil {
done <- nil
return
}
outcome := "failure"
if errors.Is(err, store.ErrConcurrencyLeaseLost) {
outcome = "lost"
if concurrencyLeaseRenewalIsFatal(err, time.Now(), leaseDeadline) {
outcome := "failure"
if errors.Is(err, store.ErrConcurrencyLeaseLost) {
outcome = "lost"
}
s.observeConcurrencyLeaseRenewal(outcome)
s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
done <- err
cancelRun()
return
}
s.observeConcurrencyLeaseRenewal(outcome)
s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
done <- err
cancelRun()
return
s.observeConcurrencyLeaseRenewal("failure")
s.logger.Warn("concurrency lease renewal failed; retrying before lease expiry",
"taskID", taskID, "leaseCount", len(leases), "outcome", "transient", "error", err)
timer.Reset(leaseRenewalRetryInterval)
continue
}
s.observeConcurrencyLeaseRenewal("success")
leaseDeadline = time.Now().Add(minimumTTL)
timer.Reset(interval)
}
}
}()
@@ -2144,6 +2160,10 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
}
}
func concurrencyLeaseRenewalIsFatal(err error, now time.Time, leaseDeadline time.Time) bool {
return errors.Is(err, store.ErrConcurrencyLeaseLost) || !now.Before(leaseDeadline)
}
func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
observer, ok := s.billingMetrics.(interface {
ObserveConcurrencyLeaseRenewal(string)
apps/api/internal/store/billing_v2_integration_test.go
+11
View File
@@ -154,6 +154,17 @@ func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testin
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
t.Fatalf("manual review task=%+v", review)
}
var attemptStatus string
var attemptErrorCode string
if err := db.pool.QueryRow(ctx, `
SELECT status, COALESCE(error_code, '')
FROM gateway_task_attempts
WHERE id=$1::uuid`, attemptID).Scan(&attemptStatus, &attemptErrorCode); err != nil {
t.Fatal(err)
}
if attemptStatus != "failed" || attemptErrorCode != "upstream_submission_unknown" {
t.Fatalf("manual review attempt status=%s error=%s", attemptStatus, attemptErrorCode)
}
var outboxStatus string
var outboxAction string
var reviewReason string
apps/api/internal/store/tasks_runtime.go
+13
View File
@@ -388,6 +388,19 @@ func markTaskExecutionManualReviewTx(
hasGatewayUser bool,
) error {
if _, err := tx.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = 'failed',
retryable = false,
error_code = 'upstream_submission_unknown',
error_message = 'upstream submission result is unknown',
finished_at = COALESCE(finished_at, now()),
upstream_submission_updated_at = now()
WHERE task_id = $1::uuid
AND status = 'running'
AND upstream_submission_status IN ('submitting', 'response_received')`, taskID); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,