BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(acceptance): 建立同构验收与弹性容量体系

Browse Source 
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
wangbo
2026-07-31 18:02:24 +08:00
parent 93d36d0e55
commit e05922b0f4
94 changed files with 12379 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/runner/service.go
+150 -87
View File
@@ -27,6 +27,8 @@ import (
type Service struct {
cfg config.Config
store *store.Store
coordinationStore *store.Store
riverStore *store.Store
logger *slog.Logger
clients map[string]clients.Client
scriptExecutor *scriptengine.Executor
@@ -95,6 +97,27 @@ func (e *TaskQueuedError) Is(target error) bool {
}
func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...billingMetricsObserver) *Service {
return NewWithStores(cfg, db, db, db, logger, observers...)
}
func NewWithCoordinationStore(
cfg config.Config,
db *store.Store,
coordinationDB *store.Store,
logger *slog.Logger,
observers ...billingMetricsObserver,
) *Service {
return NewWithStores(cfg, db, coordinationDB, db, logger, observers...)
}
func NewWithStores(
cfg config.Config,
db *store.Store,
coordinationDB *store.Store,
riverDB *store.Store,
logger *slog.Logger,
observers ...billingMetricsObserver,
) *Service {
if cfg.AsyncWorkerHardLimit == 0 {
cfg.AsyncWorkerHardLimit = 2048
}
@@ -128,10 +151,12 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
httpClients := newHTTPClientCache()
scriptExecutor := &scriptengine.Executor{Logger: logger}
service := &Service{
cfg: cfg,
store: db,
logger: logger,
scriptExecutor: scriptExecutor,
cfg: cfg,
store: db,
coordinationStore: coordinationDB,
riverStore: riverDB,
logger: logger,
scriptExecutor: scriptExecutor,
clients: map[string]clients.Client{
"openai": clients.OpenAIClient{HTTPClient: httpClients.none},
"aliyun-bailian": clients.AliyunBailianClient{HTTPClient: httpClients.none},
@@ -166,6 +191,9 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
if len(observers) > 0 {
service.billingMetrics = observers[0]
}
if service.coordinationStore == nil {
service.coordinationStore = db
}
return service
}
@@ -203,9 +231,21 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
var claimed store.GatewayTask
var err error
if distributedAdmission && !wasRunning {
claimed, err = s.store.ClaimTaskPreparation(ctx, task.ID, executionToken, taskExecutionLeaseTTL)
claimed, err = s.store.ClaimTaskPreparationForOwner(
ctx,
task.ID,
executionToken,
taskExecutionLeaseTTL,
s.workerInstanceID,
)
} else {
claimed, err = s.store.ClaimTaskExecution(ctx, task.ID, executionToken, taskExecutionLeaseTTL)
claimed, err = s.store.ClaimTaskExecutionForOwner(
ctx,
task.ID,
executionToken,
taskExecutionLeaseTTL,
s.workerInstanceID,
)
}
if err != nil {
return Result{}, err
@@ -557,6 +597,13 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
}
if hasConcurrentLimit {
if err := s.store.CheckRateLimits(ctx, s.rateLimitReservations(ctx, user, candidates[0], body)); err != nil {
if task.AsyncMode && errors.Is(err, store.ErrRateLimited) && store.RateLimitRetryable(err) {
queued, delay, queueErr := s.requeueRateLimitedTask(ctx, task, err, candidates[0])
if queueErr != nil {
return Result{}, queueErr
}
return Result{Task: queued, Output: queued.Result}, &TaskQueuedError{Delay: delay}
}
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil {
return Result{}, finishErr
@@ -577,6 +624,9 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
var alreadyAdmitted bool
admissionResult, alreadyAdmitted, admissionErr = s.activeAsyncTaskAdmission(ctx, task, plan, asyncAdmission)
if admissionErr == nil && !alreadyAdmitted {
admissionResult, admissionErr = s.tryTaskAdmission(ctx, task, plan, "")
}
if admissionErr == nil && !admissionResult.Admitted {
_ = s.store.ReleaseTaskPreparation(context.WithoutCancel(ctx), task.ID, task.ExecutionToken)
return Result{Task: task, Output: task.Result}, &TaskQueuedError{Delay: time.Second}
}
@@ -626,8 +676,13 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
}
}
}
admissionFinalized := false
if distributedAdmission {
defer s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID)
defer func() {
if !admissionFinalized {
_ = s.store.DeleteTaskAdmission(context.WithoutCancel(ctx), task.ID)
}
}()
}
firstCandidateBody := body
normalizedModelType := modelType
@@ -670,9 +725,6 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask,
}
return Result{Task: failed, Output: failed.Result}, clientErr
}
if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil {
return Result{}, err
}
var reserveErr error
walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings, reservationPricingSnapshot)
if reserveErr != nil {
@@ -872,6 +924,10 @@ candidatesLoop:
record.Metrics = mergeMetrics(record.Metrics, candidateCapabilityFilterMetrics(candidateFilterSummary))
record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log))
record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics)
callbackURL, callbackErr := s.taskCallbackURL(ctx, task.ID)
if callbackErr != nil {
return Result{}, callbackErr
}
finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{
TaskID: task.ID,
ExecutionToken: task.ExecutionToken,
@@ -891,6 +947,9 @@ candidatesLoop:
ResponseStartedAt: record.ResponseStartedAt,
ResponseFinishedAt: record.ResponseFinishedAt,
ResponseDurationMS: record.ResponseDurationMS,
CompletionCallbackURL: callbackURL,
Simulated: isSimulation(task, candidate),
FinalizeAdmission: distributedAdmission,
})
if finishErr != nil {
if errors.Is(finishErr, store.ErrTaskResultBinaryNotMaterialized) {
@@ -935,23 +994,7 @@ candidatesLoop:
return Result{}, finishErr
}
walletReservationFinalized = true
if finished.BillingStatus == "pending" {
if err := s.emit(ctx, task.ID, "task.billing.pending", "succeeded", "billing", 0.98, "task billing queued", map[string]any{
"amount": finished.FinalChargeAmount, "currency": finished.BillingCurrency,
}, isSimulation(task, candidate)); err != nil {
return Result{}, err
}
}
if err := s.emit(ctx, task.ID, "task.completed", "succeeded", "completed", 1, "task completed", map[string]any{
"result": response.Result,
"billings": billings,
"usage": record.Usage,
"metrics": record.Metrics,
"billingSummary": record.BillingSummary,
"requestId": record.RequestID,
}, isSimulation(task, candidate)); err != nil {
return Result{}, err
}
admissionFinalized = distributedAdmission
if task.AsyncMode {
// Async callers hydrate the canonical stored result through
// the task detail endpoint. The River Worker does not consume
@@ -1232,12 +1275,17 @@ func (s *Service) runCandidate(
limitResult.Leases = append(limitResult.Leases, admittedLeases...)
}
rateReservationsFinalized := false
retainAdmittedLeases := false
defer func() {
if !rateReservationsFinalized {
_ = s.store.ReleaseRateLimitReservations(context.WithoutCancel(ctx), limitResult.Reservations, "attempt_failed")
}
}()
defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.Leases)
defer func() {
if !retainAdmittedLeases {
_ = s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.Leases)
}
}()
attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
TaskID: task.ID,
@@ -1402,15 +1450,16 @@ func (s *Service) runCandidate(
var submissionWire *clients.WireResponse
runCtx, stopLeaseRenewal := s.startConcurrencyLeaseRenewal(ctx, task.ID, limitResult.Leases)
response, err := client.Run(runCtx, clients.Request{
Kind: task.Kind,
ModelType: candidate.ModelType,
Model: task.Model,
Body: providerBody,
OriginalBody: preprocessing.Input,
Candidate: candidate,
HTTPClient: requestHTTPClient,
RemoteTaskID: task.RemoteTaskID,
RemoteTaskPayload: task.RemoteTaskPayload,
Kind: task.Kind,
ModelType: candidate.ModelType,
Model: task.Model,
Body: providerBody,
OriginalBody: preprocessing.Input,
Candidate: candidate,
HTTPClient: requestHTTPClient,
RemoteTaskID: task.RemoteTaskID,
RemoteTaskPayload: task.RemoteTaskPayload,
UpstreamIdempotencyKey: task.ID,
OnRemoteTaskSubmitted: func(remoteTaskID string, payload map[string]any) error {
if strings.TrimSpace(remoteTaskID) == "" {
return nil
@@ -1446,7 +1495,8 @@ func (s *Service) runCandidate(
return nil
},
OnUpstreamResponseReceived: func() error {
return setSubmissionStatus("response_received")
submissionStatus = "response_received"
return nil
},
OnUpstreamWireResponse: func(wire *clients.WireResponse) error {
submissionWire = wire
@@ -1469,9 +1519,7 @@ func (s *Service) runCandidate(
}
callFinishedAt := time.Now()
if err == nil {
if markErr := setSubmissionStatus("response_received"); markErr != nil {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
}
submissionStatus = "response_received"
}
if response.ResponseStartedAt.IsZero() {
response.ResponseStartedAt = callStartedAt
@@ -1501,9 +1549,7 @@ func (s *Service) runCandidate(
}
if err != nil {
if clients.ErrorResponseMetadata(err).StatusCode > 0 && submissionStatus != "response_received" {
if markErr := setSubmissionStatus("response_received"); markErr != nil {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
}
submissionStatus = "response_received"
}
retryable := clients.IsRetryable(err)
requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated)
@@ -1521,16 +1567,17 @@ func (s *Service) runCandidate(
}
metrics = mergeMetrics(baseAttemptMetrics, metrics)
_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
AttemptID: attemptID,
Status: "failed",
Retryable: retryable,
RequestID: requestID,
Metrics: metrics,
ResponseStartedAt: responseStartedAt,
ResponseFinishedAt: responseFinishedAt,
ResponseDurationMS: responseDurationMS,
ErrorCode: clients.ErrorCode(err),
ErrorMessage: err.Error(),
AttemptID: attemptID,
Status: "failed",
Retryable: retryable,
UpstreamSubmissionStatus: submissionStatus,
RequestID: requestID,
Metrics: metrics,
ResponseStartedAt: responseStartedAt,
ResponseFinishedAt: responseFinishedAt,
ResponseDurationMS: responseDurationMS,
ErrorCode: clients.ErrorCode(err),
ErrorMessage: err.Error(),
})
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
if !simulated && submissionStatus == "submitting" {
@@ -1561,18 +1608,19 @@ func (s *Service) runCandidate(
"trace": []any{failureTraceEntry(err, clients.IsRetryable(err))},
})
_ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{
AttemptID: attemptID,
Status: "failed",
Retryable: clients.IsRetryable(err),
RequestID: response.RequestID,
Usage: usageToMap(response.Usage),
Metrics: metrics,
ResponseSnapshot: response.Result,
ResponseStartedAt: response.ResponseStartedAt,
ResponseFinishedAt: response.ResponseFinishedAt,
ResponseDurationMS: response.ResponseDurationMS,
ErrorCode: clients.ErrorCode(err),
ErrorMessage: err.Error(),
AttemptID: attemptID,
Status: "failed",
Retryable: clients.IsRetryable(err),
UpstreamSubmissionStatus: submissionStatus,
RequestID: response.RequestID,
Usage: usageToMap(response.Usage),
Metrics: metrics,
ResponseSnapshot: response.Result,
ResponseStartedAt: response.ResponseStartedAt,
ResponseFinishedAt: response.ResponseFinishedAt,
ResponseDurationMS: response.ResponseDurationMS,
ErrorCode: clients.ErrorCode(err),
ErrorMessage: err.Error(),
})
return clients.Response{}, err
}
@@ -1652,6 +1700,9 @@ func (s *Service) runCandidate(
}); err != nil {
s.logger.Warn("record cache affinity observation failed", "error", err, "clientId", candidate.ClientID)
}
if admittedPlatformModelID == candidate.PlatformModelID {
retainAdmittedLeases = true
}
return response, nil
}
@@ -1835,17 +1886,24 @@ func (s *Service) failTask(ctx context.Context, taskID string, executionToken st
metrics = mergeMetrics(values...)
}
metrics = s.withAttemptHistory(ctx, taskID, metrics)
callbackURL, callbackErr := s.taskCallbackURL(ctx, taskID)
if callbackErr != nil {
return store.GatewayTask{}, callbackErr
}
failed, err := s.store.FinishTaskFailure(ctx, store.FinishTaskFailureInput{
TaskID: taskID,
ExecutionToken: executionToken,
Code: code,
Message: message,
Result: buildFailureResult(code, message, requestID, cause),
RequestID: requestID,
Metrics: metrics,
ResponseStartedAt: responseStartedAt,
ResponseFinishedAt: responseFinishedAt,
ResponseDurationMS: responseDurationMS,
TaskID: taskID,
ExecutionToken: executionToken,
Code: code,
Message: message,
Result: buildFailureResult(code, message, requestID, cause),
RequestID: requestID,
Metrics: metrics,
ResponseStartedAt: responseStartedAt,
ResponseFinishedAt: responseFinishedAt,
ResponseDurationMS: responseDurationMS,
CompletionCallbackURL: callbackURL,
Simulated: simulated,
FinalizeAdmission: true,
})
if err != nil {
return store.GatewayTask{}, err
@@ -1853,9 +1911,6 @@ func (s *Service) failTask(ctx context.Context, taskID string, executionToken st
if failed.Status == "cancelled" {
return failed, nil
}
if eventErr := s.emit(ctx, taskID, "task.failed", "failed", "failed", 1, message, map[string]any{"code": code, "requestId": requestID, "metrics": metrics}, simulated); eventErr != nil {
return store.GatewayTask{}, eventErr
}
return failed, nil
}
@@ -2056,7 +2111,7 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
done <- nil
return
case <-ticker.C:
if err := s.store.RenewConcurrencyLeases(renewCtx, leases); err != nil {
if err := s.coordinationStore.RenewConcurrencyLeases(renewCtx, leases); err != nil {
if renewCtx.Err() != nil {
done <- nil
return
@@ -2113,16 +2168,10 @@ func (s *Service) withAttemptHistory(ctx context.Context, taskID string, metrics
}
func (s *Service) emit(ctx context.Context, taskID string, eventType string, status string, phase string, progress float64, message string, payload map[string]any, simulated bool) error {
acceptanceURL, acceptance, callbackErr := s.store.AcceptanceCallbackURL(ctx, taskID)
callbackURL, callbackErr := s.taskCallbackURL(ctx, taskID)
if callbackErr != nil {
return callbackErr
}
callbackURL := ""
if acceptance && acceptanceURL != "" {
callbackURL = acceptanceURL
} else if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
callbackURL = s.cfg.TaskProgressCallbackURL
}
event, err := s.store.AddTaskEventWithCallback(
ctx,
taskID,
@@ -2144,6 +2193,20 @@ func (s *Service) emit(ctx context.Context, taskID string, eventType string, sta
return nil
}
func (s *Service) taskCallbackURL(ctx context.Context, taskID string) (string, error) {
acceptanceURL, acceptance, err := s.store.AcceptanceCallbackURL(ctx, taskID)
if err != nil {
return "", err
}
if acceptance && acceptanceURL != "" {
return acceptanceURL, nil
}
if s.cfg.TaskProgressCallbackEnabled && s.cfg.TaskProgressCallbackURL != "" {
return s.cfg.TaskProgressCallbackURL, nil
}
return "", nil
}
func modelTypeFromKind(kind string, body map[string]any) string {
if requested := requestedModelTypeFromBody(body); requested != "" {
return requested