增加单一源保护全局策略
This commit is contained in:
@@ -240,6 +240,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
|
||||
}
|
||||
maxPlatforms := maxPlatformsForCandidates(candidates, runnerPolicy)
|
||||
maxFailoverDuration := maxFailoverDurationForCandidates(candidates, runnerPolicy)
|
||||
singleSourceProtected := singleSourceProtectionActive(candidates, maxPlatforms, runnerPolicy)
|
||||
var lastErr error
|
||||
var lastCandidate store.RuntimeModelCandidate
|
||||
var lastPreprocessing *parameterPreprocessingLog
|
||||
@@ -275,7 +276,7 @@ candidatesLoop:
|
||||
break candidatesLoop
|
||||
}
|
||||
candidateBody := preprocessing.Body
|
||||
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta)
|
||||
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, singleSourceProtected)
|
||||
if err == nil {
|
||||
attemptNo = nextAttemptNo
|
||||
billings := s.billings(ctx, user, task.Kind, candidateBody, candidate, response, isSimulation(task, candidate))
|
||||
@@ -445,7 +446,7 @@ candidatesLoop:
|
||||
if !decision.Retry {
|
||||
break
|
||||
}
|
||||
s.applyFailoverAction(ctx, task.ID, candidate, decision, isSimulation(task, candidate))
|
||||
s.applyFailoverAction(ctx, task.ID, candidate, decision, isSimulation(task, candidate), singleSourceProtected)
|
||||
if err := s.emit(ctx, task.ID, "task.retrying", "running", "retry", 0.55, "retrying next client", addPolicyTracePayload(map[string]any{
|
||||
"attempt": attemptNo,
|
||||
"action": decision.Action,
|
||||
@@ -486,7 +487,7 @@ candidatesLoop:
|
||||
return Result{Task: failed, Output: failed.Result}, lastErr
|
||||
}
|
||||
|
||||
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta) (clients.Response, error) {
|
||||
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, singleSourceProtected bool) (clients.Response, error) {
|
||||
simulated := isSimulation(task, candidate)
|
||||
baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing))
|
||||
reservations := s.rateLimitReservations(ctx, user, candidate, body)
|
||||
@@ -638,7 +639,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
|
||||
ErrorMessage: err.Error(),
|
||||
})
|
||||
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
|
||||
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated)
|
||||
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected)
|
||||
return clients.Response{}, err
|
||||
}
|
||||
uploadedResult, err := s.uploadGeneratedAssets(ctx, task.ID, task.Kind, response.Result)
|
||||
@@ -1099,6 +1100,42 @@ func maxPlatformsForCandidates(candidates []store.RuntimeModelCandidate, runnerP
|
||||
return maxPlatforms
|
||||
}
|
||||
|
||||
func singleSourceProtectionActive(candidates []store.RuntimeModelCandidate, maxPlatforms int, runnerPolicy store.RunnerPolicy) bool {
|
||||
if !runnerSingleSourceProtectionEnabled(runnerPolicy) {
|
||||
return false
|
||||
}
|
||||
return runtimeCandidateSourceCount(candidates, maxPlatforms) <= 1
|
||||
}
|
||||
|
||||
func runnerSingleSourceProtectionEnabled(runnerPolicy store.RunnerPolicy) bool {
|
||||
if enabled, ok := runnerPolicy.SingleSourcePolicy["enabled"].(bool); ok {
|
||||
return enabled
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func runtimeCandidateSourceCount(candidates []store.RuntimeModelCandidate, maxPlatforms int) int {
|
||||
limit := len(candidates)
|
||||
if maxPlatforms > 0 && maxPlatforms < limit {
|
||||
limit = maxPlatforms
|
||||
}
|
||||
sources := make(map[string]bool, limit)
|
||||
for index := 0; index < limit; index++ {
|
||||
key := strings.TrimSpace(candidates[index].PlatformID)
|
||||
if key == "" {
|
||||
key = strings.TrimSpace(candidates[index].PlatformModelID)
|
||||
}
|
||||
if key == "" {
|
||||
key = strings.TrimSpace(candidates[index].ClientID)
|
||||
}
|
||||
if key == "" {
|
||||
key = strconv.Itoa(index)
|
||||
}
|
||||
sources[key] = true
|
||||
}
|
||||
return len(sources)
|
||||
}
|
||||
|
||||
func maxFailoverDurationForCandidates(candidates []store.RuntimeModelCandidate, runnerPolicy store.RunnerPolicy) time.Duration {
|
||||
seconds := intFromPolicy(runnerPolicy.FailoverPolicy, "maxDurationSeconds")
|
||||
if seconds <= 0 {
|
||||
|
||||
Reference in New Issue
Block a user