fix(runner): 修复图像结果转存与任务租约失效

修正非语义字段长 Base64 元数据被误判为媒体的问题,并允许按文件签名识别真实媒体后转存 OSS;同时将网关本地存储失败与上游失败分离,避免重复调用和错误降权。

异步任务复用已分配并发名额前立即刷新租约,租约已丢失时重新排队获取,避免排队耗时侵蚀租约后错误调用上游。生产环境关闭 server-main 尚未实现的进度回调,保留 Gateway 任务详情与事件查询。

验证:API 全量 go test、go vet、gofmt、Kubernetes 渲染、迁移安全检查及真实阿里云 OSS 读写/生命周期验收通过。
This commit is contained in:
2026-08-04 13:48:44 +08:00
parent 6a64d3936a
commit 394cc6288f
8 changed files with 127 additions and 18 deletions
+10
View File
@@ -338,6 +338,16 @@ func (s *Service) activeAsyncTaskAdmission(
if len(leases) == 0 { if len(leases) == 0 {
return store.TaskAdmissionResult{}, false, nil return store.TaskAdmissionResult{}, false, nil
} }
leaseStore := s.coordinationStore
if leaseStore == nil {
leaseStore = s.store
}
if err := leaseStore.RenewConcurrencyLeases(ctx, leases); err != nil {
if errors.Is(err, store.ErrConcurrencyLeaseLost) {
return store.TaskAdmissionResult{}, false, nil
}
return store.TaskAdmissionResult{}, false, err
}
return store.TaskAdmissionResult{ return store.TaskAdmissionResult{
Admission: *admission, Admission: *admission,
Admitted: true, Admitted: true,
+8 -1
View File
@@ -517,7 +517,14 @@ func localBinaryStringBytes(key string, value string, siblings map[string]any) (
if err != nil || len(payload) == 0 { if err != nil || len(payload) == 0 {
return nil, "", "", false return nil, "", "", false
} }
return payload, firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key)), "raw", true contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
if !strict && contentType == "" {
contentType = detectGeneratedAssetContentType(payload)
if !generatedContentTypeIsMedia(contentType) && !generatedContentTypeIsDocument(contentType) {
return nil, "", "", false
}
}
return payload, contentType, "raw", true
} }
func localBufferObjectBytes(value map[string]any) ([]byte, string, bool) { func localBufferObjectBytes(value map[string]any) ([]byte, string, bool) {
@@ -1,6 +1,7 @@
package runner package runner
import ( import (
"bytes"
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/base64" "encoding/base64"
@@ -311,6 +312,24 @@ func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.
} }
} }
func TestTaskResultIgnoresLongOpaqueBase64Metadata(t *testing.T) {
opaque := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("opaque metadata ", 400)))
result := map[string]any{"thought_signature": opaque}
if TaskResultHasInlineBinary(result) {
t.Fatal("opaque Base64 metadata must not be mistaken for generated media")
}
}
func TestTaskResultDetectsLongBase64MediaWithoutSemanticKey(t *testing.T) {
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 4096)...)
result := map[string]any{"provider_payload": base64.StdEncoding.EncodeToString(payload)}
if !TaskResultHasInlineBinary(result) {
t.Fatal("signature-detected generated media must still be materialized")
}
}
func bytesToAny(payload []byte) []any { func bytesToAny(payload []byte) []any {
result := make([]any, len(payload)) result := make([]any, len(payload))
for index, value := range payload { for index, value := range payload {
+11 -9
View File
@@ -303,16 +303,16 @@ func priorityDemoteDecisionForCandidate(runnerPolicy store.RunnerPolicy, err err
} }
func isResultPersistenceFailure(err error) bool { func isResultPersistenceFailure(err error) bool {
switch strings.ToLower(strings.TrimSpace(clients.ErrorCode(err))) { return isGatewayResultPersistenceCode(clients.ErrorCode(err))
case "local_result_storage_unavailable",
"binary_result_too_large",
"binary_result_corrupted",
"binary_result_expired",
"result_binary_not_materialized":
return true
default:
return false
} }
func isGatewayResultPersistenceCode(code string) bool {
code = strings.ToLower(strings.TrimSpace(code))
return strings.HasPrefix(code, "storage_") ||
strings.HasPrefix(code, "upload_") ||
strings.HasPrefix(code, "binary_result_") ||
strings.HasPrefix(code, "local_result_") ||
strings.HasPrefix(code, "result_binary_")
} }
func effectiveFailoverPolicy(base map[string]any, override map[string]any) map[string]any { func effectiveFailoverPolicy(base map[string]any, override map[string]any) map[string]any {
@@ -353,6 +353,8 @@ func failureCategory(code string, status int, message string) string {
switch { switch {
case code == "insufficient_balance": case code == "insufficient_balance":
return "insufficient_balance" return "insufficient_balance"
case isGatewayResultPersistenceCode(code):
return "gateway_storage"
case code == "rate_limit" || status == 429: case code == "rate_limit" || status == 429:
return "rate_limit" return "rate_limit"
case code == "network": case code == "network":
@@ -495,3 +495,37 @@ func TestResolveCandidateFailureUnmatchedRetryableUsesSameThenNext(t *testing.T)
t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision) t.Fatalf("unmatched exhausted error should rotate without side effects, got %+v", decision)
} }
} }
func TestGatewayStorageFailureNeverRetriesProvider(t *testing.T) {
err := &clients.ClientError{
Code: "storage_write_failed",
Message: "generated binary result could not be written to object storage",
StatusCode: 503,
Retryable: true,
}
candidate := store.RuntimeModelCandidate{ModelRetryPolicy: map[string]any{
"enabled": true,
"allowKeywords": []any{"5xx"},
}}
retry := retryDecisionForCandidate(candidate, err)
if retry.Retry || retry.Reason != "result_persistence_failed" {
t.Fatalf("Gateway storage failure must not repeat an upstream request: %+v", retry)
}
failover := failoverDecisionForCandidate(store.RunnerPolicy{
Status: "active",
FailoverPolicy: map[string]any{"enabled": true, "allowCategories": []any{"provider_5xx"}},
}, candidate, err)
if failover.Retry || failover.Reason != "result_persistence_failed" {
t.Fatalf("Gateway storage failure must not fail over to another provider: %+v", failover)
}
decision := resolveCandidateFailure(resolveCandidateFailureInput{
RunnerPolicy: store.RunnerPolicy{Status: "active"},
Err: err,
HasNextCandidate: true,
Async: true,
})
if decision.Route != "stop" || decision.Reason != "result_persistence_failed" || decision.Info.Category != "gateway_storage" {
t.Fatalf("Gateway storage failure classification is incorrect: %+v", decision)
}
}
+4 -6
View File
@@ -423,18 +423,16 @@ func generatedRawInlineMediaAsset(key string, value string, siblings map[string]
} }
keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key) keyLooksLikeMediaPayload := generatedRawDataMediaPayloadKey(key)
contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key)) contentType := firstNonEmptyString(mediaContentTypeFromItem(siblings), defaultContentTypeForRawMediaKey(key))
if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) { if !keyLooksLikeMediaPayload && !generatedContentTypeIsMedia(contentType) && !generatedContentTypeIsDocument(contentType) &&
return nil, false !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < localBinaryGenericBase64MinLength {
}
if !keyLooksLikeMediaPayload && !strings.HasPrefix(strings.ToLower(raw), "data:") && len(raw) < 128 {
return nil, false return nil, false
} }
payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload) payload, payloadContentType, ok, err := inlineMediaPayload(raw, keyLooksLikeMediaPayload)
if err != nil || !ok || len(payload) == 0 { if err != nil || !ok || len(payload) == 0 {
return nil, false return nil, false
} }
contentType = firstNonEmptyString(payloadContentType, contentType) contentType = firstNonEmptyString(payloadContentType, contentType, detectGeneratedAssetContentType(payload))
if !generatedContentTypeIsMedia(contentType) { if !generatedContentTypeIsMedia(contentType) && !generatedContentTypeIsDocument(contentType) {
return nil, false return nil, false
} }
kind := mediaKindForAsset(taskKind, siblings, key, contentType) kind := mediaKindForAsset(taskKind, siblings, key, contentType)
+37
View File
@@ -338,6 +338,43 @@ func TestFinalizeGeneratedAssetsUploadsNestedInlineBinaryUnderDefaultPolicy(t *t
} }
} }
func TestFinalizeGeneratedAssetsMaterializesDetectedMediaAndKeepsOpaqueMetadata(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
defer server.Close()
service := &Service{}
channels := []store.FileStorageChannel{testObjectStorageChannel("s3", server.URL, "s3-result")}
payload := append([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, bytes.Repeat([]byte{0}, 4096)...)
opaque := base64.StdEncoding.EncodeToString([]byte(strings.Repeat("opaque metadata ", 400)))
result := map[string]any{
"provider_payload": base64.StdEncoding.EncodeToString(payload),
"thought_signature": opaque,
}
finalized, err := service.finalizeGeneratedAssets(
t.Context(),
"task-provider-payload",
"images.generations",
result,
defaultGeneratedAssetUploadPolicy(),
channels,
true,
0,
)
if err != nil {
t.Fatal(err)
}
if TaskResultHasInlineBinary(finalized) {
t.Fatalf("finalized result still contains generated media: %#v", finalized)
}
reference, ok := finalized["provider_payload"].(map[string]any)
if !ok || reference["assetRef"] == nil || reference["upload"] == nil {
t.Fatalf("detected media was not objectified: %#v", finalized["provider_payload"])
}
if finalized["thought_signature"] != opaque {
t.Fatal("opaque provider metadata must remain unchanged")
}
}
func TestResolvedGeneratedAssetContentTypePrefersDetectedMedia(t *testing.T) { func TestResolvedGeneratedAssetContentTypePrefersDetectedMedia(t *testing.T) {
pngPayload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0} pngPayload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0}
@@ -15,7 +15,9 @@ data:
IDENTITY_SECURITY_EVENTS_STALE_AFTER_SECONDS: "180" IDENTITY_SECURITY_EVENTS_STALE_AFTER_SECONDS: "180"
IDENTITY_SECURITY_EVENTS_CLOCK_SKEW_SECONDS: "60" IDENTITY_SECURITY_EVENTS_CLOCK_SKEW_SECONDS: "60"
SERVER_MAIN_BASE_URL: http://10.77.0.1:3001 SERVER_MAIN_BASE_URL: http://10.77.0.1:3001
TASK_PROGRESS_CALLBACK_ENABLED: "true" # server-main 当前没有实现该回调路由。关闭无效投递,任务结果继续通过
# Gateway task detail/events 接口读取;实现并验收接收端后再显式开启。
TASK_PROGRESS_CALLBACK_ENABLED: "false"
TASK_PROGRESS_CALLBACK_URL: http://10.77.0.1:3001/internal/platform/task-progress-callbacks TASK_PROGRESS_CALLBACK_URL: http://10.77.0.1:3001/internal/platform/task-progress-callbacks
TASK_PROGRESS_CALLBACK_TIMEOUT_MS: "5000" TASK_PROGRESS_CALLBACK_TIMEOUT_MS: "5000"
TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS: "10" TASK_PROGRESS_CALLBACK_MAX_ATTEMPTS: "10"