wangbo
0f0998cbcf
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。 将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。 引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。 验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
329 lines
12 KiB
Go
329 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func newLocalBinaryTestService(t *testing.T) *Service {
|
|
t.Helper()
|
|
return &Service{cfg: config.Config{
|
|
LocalGeneratedStorageDir: t.TempDir(),
|
|
LocalResultTTLHours: 24,
|
|
LocalResultMinFreeBytes: 1,
|
|
LocalResultMaxBytes: 1024 * 1024,
|
|
LocalResultMaxTaskBytes: 2 * 1024 * 1024,
|
|
}}
|
|
}
|
|
|
|
func writeHistoricalLocalBinaryFixture(t *testing.T, service *Service, taskID string, payload []byte) string {
|
|
t.Helper()
|
|
digest := sha256.Sum256(payload)
|
|
taskDir := filepath.Join(service.localBinaryResultRoot(), safeLocalBinaryTaskDir(taskID))
|
|
if err := os.MkdirAll(taskDir, 0o750); err != nil {
|
|
t.Fatalf("create historical result fixture directory: %v", err)
|
|
}
|
|
path := filepath.Join(taskDir, hex.EncodeToString(digest[:])+".bin")
|
|
if err := os.WriteFile(path, payload, 0o640); err != nil {
|
|
t.Fatalf("write historical result fixture: %v", err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func TestHydrateHistoricalLocalBinaryResult(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
payload := []byte("one binary result shared across representations")
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
input := map[string]any{
|
|
"data": []any{
|
|
map[string]any{
|
|
"b64_json": encoded,
|
|
"data_uri": "data:image/png;base64," + encoded,
|
|
"buffer": map[string]any{
|
|
"type": "Buffer",
|
|
"data": bytesToAny(payload),
|
|
"mimeType": "image/png",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-123", input, false, true)
|
|
if err != nil {
|
|
t.Fatalf("materialize local binary result: %v", err)
|
|
}
|
|
if !changed {
|
|
t.Fatal("expected binary result to be materialized")
|
|
}
|
|
item := persistent["data"].([]any)[0].(map[string]any)
|
|
for _, key := range []string{"b64_json", "data_uri", "buffer"} {
|
|
placeholder, ok := item[key].(string)
|
|
if !ok || !strings.HasPrefix(placeholder, localBinaryPlaceholderPrefix) {
|
|
t.Fatalf("%s was not replaced with a placeholder: %+v", key, item[key])
|
|
}
|
|
if len(placeholder) > 200 {
|
|
t.Fatalf("%s placeholder exceeds 200 bytes: %d", key, len(placeholder))
|
|
}
|
|
}
|
|
if input["data"].([]any)[0].(map[string]any)["b64_json"] != encoded {
|
|
t.Fatal("materializer mutated the provider result")
|
|
}
|
|
persistentJSON, err := json.Marshal(persistent)
|
|
if err != nil {
|
|
t.Fatalf("marshal persistent result: %v", err)
|
|
}
|
|
if len(persistentJSON) > 32*1024 {
|
|
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
|
|
}
|
|
writeHistoricalLocalBinaryFixture(t, service, "task-123", payload)
|
|
|
|
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
|
|
entries, err := os.ReadDir(taskDir)
|
|
if err != nil {
|
|
t.Fatalf("read local result dir: %v", err)
|
|
}
|
|
if len(entries) != 1 {
|
|
t.Fatalf("same payload should reuse one local file, got %d", len(entries))
|
|
}
|
|
taskInfo, err := os.Stat(taskDir)
|
|
if err != nil {
|
|
t.Fatalf("stat task directory: %v", err)
|
|
}
|
|
if taskInfo.Mode().Perm() != 0o750 {
|
|
t.Fatalf("task directory mode = %v, want 0750", taskInfo.Mode().Perm())
|
|
}
|
|
fileInfo, err := entries[0].Info()
|
|
if err != nil {
|
|
t.Fatalf("stat result file: %v", err)
|
|
}
|
|
if fileInfo.Mode().Perm() != 0o640 {
|
|
t.Fatalf("result file mode = %v, want 0640", fileInfo.Mode().Perm())
|
|
}
|
|
|
|
wire, err := service.HydrateTaskResult(context.Background(), "task-123", persistent)
|
|
if err != nil {
|
|
t.Fatalf("hydrate local binary result: %v", err)
|
|
}
|
|
wireItem := wire["data"].([]any)[0].(map[string]any)
|
|
if wireItem["b64_json"] != encoded {
|
|
t.Fatalf("raw Base64 mismatch: got %v", wireItem["b64_json"])
|
|
}
|
|
if wireItem["data_uri"] != "data:image/png;base64,"+encoded {
|
|
t.Fatalf("data URI mismatch: got %v", wireItem["data_uri"])
|
|
}
|
|
if wireItem["buffer"] != encoded {
|
|
t.Fatalf("Buffer should hydrate as Base64: got %v", wireItem["buffer"])
|
|
}
|
|
}
|
|
|
|
func TestHydrateGeneratedResultAssetReference(t *testing.T) {
|
|
payload := []byte("verified generated image bytes")
|
|
digest := sha256.Sum256(payload)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "image/png")
|
|
_, _ = w.Write(payload)
|
|
}))
|
|
defer server.Close()
|
|
|
|
reference := func(hash string) map[string]any {
|
|
return map[string]any{
|
|
"assetRef": map[string]any{
|
|
"sha256": hash,
|
|
"contentType": "image/png",
|
|
"size": len(payload),
|
|
"url": server.URL + "/generated.png",
|
|
},
|
|
"assetStorage": map[string]any{
|
|
"scene": store.FileStorageSceneImageResult,
|
|
"source": "b64_json",
|
|
},
|
|
}
|
|
}
|
|
service := &Service{}
|
|
result := map[string]any{"data": []any{map[string]any{"b64_json": reference(hex.EncodeToString(digest[:]))}}}
|
|
hydrated, err := service.HydrateTaskResult(context.Background(), "task-remote", result)
|
|
if err != nil {
|
|
t.Fatalf("hydrate generated result asset: %v", err)
|
|
}
|
|
item := hydrated["data"].([]any)[0].(map[string]any)
|
|
if got, want := item["b64_json"], base64.StdEncoding.EncodeToString(payload); got != want {
|
|
t.Fatalf("hydrated Base64=%v, want %v", got, want)
|
|
}
|
|
|
|
result = map[string]any{"data": []any{map[string]any{"b64_json": reference(strings.Repeat("0", 64))}}}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-corrupted", result)
|
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
|
}
|
|
|
|
func TestHydrateGeneratedResultRefreshesPrivateObjectURL(t *testing.T) {
|
|
cfg := config.Config{
|
|
MediaOSSDirectEnabled: true,
|
|
MediaOSSEndpoint: "https://oss.example.com",
|
|
MediaOSSBucket: "media-bucket",
|
|
MediaOSSAccessKeyID: "access-id",
|
|
MediaOSSAccessKeySecret: "access-secret",
|
|
MediaOSSObjectPrefix: "media",
|
|
}
|
|
service := &Service{cfg: cfg, directOSS: newDirectOSSUploader(cfg)}
|
|
result := map[string]any{
|
|
"data": []any{map[string]any{
|
|
"url": "https://expired.example.com/result.png",
|
|
"image_url": "https://expired.example.com/result.png",
|
|
"upload": map[string]any{
|
|
"url": "https://expired.example.com/result.png",
|
|
"objectKey": "media/image_result/2026/08/04/hash.png",
|
|
"accessScope": "private",
|
|
"storageChannel": map[string]any{
|
|
"channelKey": "environment-direct-oss",
|
|
"provider": "aliyun_oss",
|
|
},
|
|
},
|
|
}},
|
|
}
|
|
hydrated, err := service.HydrateTaskResult(t.Context(), "task-private-url", result)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
item := hydrated["data"].([]any)[0].(map[string]any)
|
|
for _, key := range []string{"url", "image_url"} {
|
|
value := stringFromAny(item[key])
|
|
if !strings.Contains(value, "OSSAccessKeyId=access-id") || strings.Contains(value, "expired.example.com") {
|
|
t.Fatalf("%s was not refreshed: %q", key, value)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultTTLHours = 1
|
|
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
|
|
persistent, _, err := service.transformLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded}, false, true)
|
|
if err != nil {
|
|
t.Fatalf("materialize fixture: %v", err)
|
|
}
|
|
path := writeHistoricalLocalBinaryFixture(t, service, "task-expired", []byte("expiring result"))
|
|
old := time.Now().Add(-2 * time.Hour)
|
|
if err := os.Chtimes(path, old, old); err != nil {
|
|
t.Fatalf("age fixture: %v", err)
|
|
}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
|
|
assertClientErrorCode(t, err, "binary_result_expired")
|
|
|
|
now := time.Now()
|
|
if err := os.Chtimes(path, now, now); err != nil {
|
|
t.Fatalf("refresh fixture: %v", err)
|
|
}
|
|
if err := os.WriteFile(path, []byte("tampered"), 0o640); err != nil {
|
|
t.Fatalf("tamper fixture: %v", err)
|
|
}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
|
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
|
}
|
|
|
|
func TestHistoricalLocalBinaryFixtureEnforcesLimitsAndKeepsText(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMaxBytes = 4
|
|
_, _, err := service.transformLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
|
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
|
|
}, false, true)
|
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
|
|
|
persistent, changed, err := service.transformLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
|
"message": "YWJjZA==",
|
|
}, false, true)
|
|
if err != nil || changed || persistent["message"] != "YWJjZA==" {
|
|
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
|
|
}
|
|
|
|
service = newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMaxBytes = 1024
|
|
service.cfg.LocalResultMaxTaskBytes = 8
|
|
_, _, err = service.transformLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
|
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
|
|
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
|
|
}, false, true)
|
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
|
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestCompactExpiredTaskResultUsesObjectStorageAndDoesNotWriteFiles(t *testing.T) {
|
|
var uploads int
|
|
storageServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != http.MethodPut {
|
|
t.Fatalf("method=%s, want PUT", r.Method)
|
|
}
|
|
uploads++
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer storageServer.Close()
|
|
service := newLocalBinaryTestService(t)
|
|
service.directOSS = &directOSSUploader{
|
|
endpoint: storageServer.URL, bucket: "bucket", accessKeyID: "access-id", accessKeySecret: "access-secret", objectPrefix: "media",
|
|
}
|
|
service.cfg.LocalResultMaxBytes = 1
|
|
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
|
|
|
|
persistent, changed, err := service.CompactExpiredTaskResultForStorage(context.Background(), "task-old", map[string]any{
|
|
"b64_json": encoded,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("compact expired result: %v", err)
|
|
}
|
|
if !changed || localBinaryResultHasPlaceholders(persistent) || TaskResultHasInlineBinary(persistent) {
|
|
t.Fatalf("expired result was not objectified: %+v", persistent)
|
|
}
|
|
if uploads != 1 {
|
|
t.Fatalf("object storage uploads=%d, want 1", uploads)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("expired compaction must not create a task directory: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestHistoricalLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
|
err := localBinaryStorageError(errors.New("write failed"))
|
|
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
|
if clients.IsRetryable(err) {
|
|
t.Fatal("local result storage failure must not call the provider again")
|
|
}
|
|
if retryDecisionForCandidate(store.RuntimeModelCandidate{}, err).Retry {
|
|
t.Fatal("local result storage failure must not retry the same provider client")
|
|
}
|
|
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
|
|
t.Fatal("local result storage failure must not fail over to another provider")
|
|
}
|
|
}
|
|
|
|
func bytesToAny(payload []byte) []any {
|
|
result := make([]any, len(payload))
|
|
for index, value := range payload {
|
|
result[index] = float64(value)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func assertClientErrorCode(t *testing.T, err error, code string) {
|
|
t.Helper()
|
|
var clientErr *clients.ClientError
|
|
if !errors.As(err, &clientErr) || clientErr.Code != code {
|
|
t.Fatalf("error = %v, want client error code %s", err, code)
|
|
}
|
|
}
|