fix(runner): record failed task attempts

This commit is contained in:
2026-05-17 20:50:20 +08:00
parent ae197a742f
commit 90c3315468
4 changed files with 287 additions and 29 deletions
+45
View File
@@ -1,6 +1,7 @@
package runner
import (
"errors"
"strings"
"time"
@@ -210,6 +211,9 @@ func failureMetrics(err error, simulated bool) (string, map[string]any, time.Tim
metrics["error"] = err.Error()
metrics["errorCategory"] = info.Category
metrics["retryable"] = retryable
if detail := rateLimitFailureDetail(err); len(detail) > 0 {
metrics["rateLimit"] = detail
}
}
if meta.StatusCode > 0 {
metrics["statusCode"] = meta.StatusCode
@@ -226,6 +230,47 @@ func failureMetrics(err error, simulated bool) (string, map[string]any, time.Tim
return meta.RequestID, metrics, meta.ResponseStartedAt, meta.ResponseFinishedAt, meta.ResponseDurationMS
}
func rateLimitFailureDetail(err error) map[string]any {
var limitErr *store.RateLimitExceededError
if !errors.As(err, &limitErr) {
return nil
}
detail := map[string]any{
"scopeType": limitErr.ScopeType,
"scopeKey": limitErr.ScopeKey,
"scopeName": limitErr.ScopeName,
"metric": limitErr.Metric,
"limit": limitErr.Limit,
"amount": limitErr.Amount,
"current": limitErr.Current,
"used": limitErr.Used,
"reserved": limitErr.Reserved,
"projected": limitErr.Projected,
"windowSeconds": limitErr.WindowSeconds,
"retryable": limitErr.Retryable,
"exceeded": map[string]any{
"metric": limitErr.Metric,
"current": limitErr.Current,
"amount": limitErr.Amount,
"projected": limitErr.Projected,
"limit": limitErr.Limit,
},
}
if limitErr.RetryAfter > 0 {
detail["retryAfterMs"] = limitErr.RetryAfter.Milliseconds()
}
if !limitErr.ResetAt.IsZero() {
detail["resetAt"] = limitErr.ResetAt.UTC().Format(time.RFC3339Nano)
}
if len(limitErr.ScopeMetadata) > 0 {
detail["scopeMetadata"] = limitErr.ScopeMetadata
}
if len(limitErr.Policy) > 0 {
detail["rateLimitPolicy"] = limitErr.Policy
}
return detail
}
func mergeMetrics(values ...map[string]any) map[string]any {
out := map[string]any{}
for _, value := range values {