wangbo
4af86b22ee
将非流式 Gemini generateContent 改为 River Worker 执行，API 通过批量状态查询等待完成，避免高并发同步请求耗尽 API 数据库连接池。\n\n生成图片在持久化时保存带哈希的资产引用，响应恢复时下载并校验后重建 Base64，数据库不保存媒体原文。请求体物化增加前置内存门禁，并扩展双 API/Worker PostgreSQL 压力测试。\n\n验证：go test ./... -count=1；go vet ./...；pnpm openapi；迁移安全检查；64 与 256 请求双角色异步 Gemini 压力测试。
277 lines
10 KiB
Go
277 lines
10 KiB
Go
package runner
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
func newLocalBinaryTestService(t *testing.T) *Service {
|
|
t.Helper()
|
|
return &Service{cfg: config.Config{
|
|
LocalGeneratedStorageDir: t.TempDir(),
|
|
LocalResultTTLHours: 24,
|
|
LocalResultMinFreeBytes: 1,
|
|
LocalResultMaxBytes: 1024 * 1024,
|
|
LocalResultMaxTaskBytes: 2 * 1024 * 1024,
|
|
}}
|
|
}
|
|
|
|
func TestMaterializeAndHydrateLocalBinaryResult(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
payload := []byte("one binary result shared across representations")
|
|
encoded := base64.StdEncoding.EncodeToString(payload)
|
|
input := map[string]any{
|
|
"data": []any{
|
|
map[string]any{
|
|
"b64_json": encoded,
|
|
"data_uri": "data:image/png;base64," + encoded,
|
|
"buffer": map[string]any{
|
|
"type": "Buffer",
|
|
"data": bytesToAny(payload),
|
|
"mimeType": "image/png",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-123", input)
|
|
if err != nil {
|
|
t.Fatalf("materialize local binary result: %v", err)
|
|
}
|
|
if !changed {
|
|
t.Fatal("expected binary result to be materialized")
|
|
}
|
|
item := persistent["data"].([]any)[0].(map[string]any)
|
|
for _, key := range []string{"b64_json", "data_uri", "buffer"} {
|
|
placeholder, ok := item[key].(string)
|
|
if !ok || !strings.HasPrefix(placeholder, localBinaryPlaceholderPrefix) {
|
|
t.Fatalf("%s was not replaced with a placeholder: %+v", key, item[key])
|
|
}
|
|
if len(placeholder) > 200 {
|
|
t.Fatalf("%s placeholder exceeds 200 bytes: %d", key, len(placeholder))
|
|
}
|
|
}
|
|
if input["data"].([]any)[0].(map[string]any)["b64_json"] != encoded {
|
|
t.Fatal("materializer mutated the provider result")
|
|
}
|
|
persistentJSON, err := json.Marshal(persistent)
|
|
if err != nil {
|
|
t.Fatalf("marshal persistent result: %v", err)
|
|
}
|
|
if len(persistentJSON) > 32*1024 {
|
|
t.Fatalf("binary-only persistent result exceeds 32KiB: %d", len(persistentJSON))
|
|
}
|
|
|
|
taskDir := filepath.Join(service.localBinaryResultRoot(), "task-123")
|
|
entries, err := os.ReadDir(taskDir)
|
|
if err != nil {
|
|
t.Fatalf("read local result dir: %v", err)
|
|
}
|
|
if len(entries) != 1 {
|
|
t.Fatalf("same payload should reuse one local file, got %d", len(entries))
|
|
}
|
|
taskInfo, err := os.Stat(taskDir)
|
|
if err != nil {
|
|
t.Fatalf("stat task directory: %v", err)
|
|
}
|
|
if taskInfo.Mode().Perm() != 0o750 {
|
|
t.Fatalf("task directory mode = %v, want 0750", taskInfo.Mode().Perm())
|
|
}
|
|
fileInfo, err := entries[0].Info()
|
|
if err != nil {
|
|
t.Fatalf("stat result file: %v", err)
|
|
}
|
|
if fileInfo.Mode().Perm() != 0o640 {
|
|
t.Fatalf("result file mode = %v, want 0640", fileInfo.Mode().Perm())
|
|
}
|
|
|
|
wire, err := service.HydrateTaskResult(context.Background(), "task-123", persistent)
|
|
if err != nil {
|
|
t.Fatalf("hydrate local binary result: %v", err)
|
|
}
|
|
wireItem := wire["data"].([]any)[0].(map[string]any)
|
|
if wireItem["b64_json"] != encoded {
|
|
t.Fatalf("raw Base64 mismatch: got %v", wireItem["b64_json"])
|
|
}
|
|
if wireItem["data_uri"] != "data:image/png;base64,"+encoded {
|
|
t.Fatalf("data URI mismatch: got %v", wireItem["data_uri"])
|
|
}
|
|
if wireItem["buffer"] != encoded {
|
|
t.Fatalf("Buffer should hydrate as Base64: got %v", wireItem["buffer"])
|
|
}
|
|
}
|
|
|
|
func TestHydrateGeneratedResultAssetReference(t *testing.T) {
|
|
payload := []byte("verified generated image bytes")
|
|
digest := sha256.Sum256(payload)
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "image/png")
|
|
_, _ = w.Write(payload)
|
|
}))
|
|
defer server.Close()
|
|
|
|
reference := func(hash string) map[string]any {
|
|
return map[string]any{
|
|
"assetRef": map[string]any{
|
|
"sha256": hash,
|
|
"contentType": "image/png",
|
|
"size": len(payload),
|
|
"url": server.URL + "/generated.png",
|
|
},
|
|
"assetStorage": map[string]any{
|
|
"scene": store.FileStorageSceneImageResult,
|
|
"source": "b64_json",
|
|
},
|
|
}
|
|
}
|
|
service := &Service{}
|
|
result := map[string]any{"data": []any{map[string]any{"b64_json": reference(hex.EncodeToString(digest[:]))}}}
|
|
hydrated, err := service.HydrateTaskResult(context.Background(), "task-remote", result)
|
|
if err != nil {
|
|
t.Fatalf("hydrate generated result asset: %v", err)
|
|
}
|
|
item := hydrated["data"].([]any)[0].(map[string]any)
|
|
if got, want := item["b64_json"], base64.StdEncoding.EncodeToString(payload); got != want {
|
|
t.Fatalf("hydrated Base64=%v, want %v", got, want)
|
|
}
|
|
|
|
result = map[string]any{"data": []any{map[string]any{"b64_json": reference(strings.Repeat("0", 64))}}}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-corrupted", result)
|
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
|
}
|
|
|
|
func TestHydrateLocalBinaryResultReturnsExpiredAndCorruptedErrors(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultTTLHours = 1
|
|
encoded := base64.StdEncoding.EncodeToString([]byte("expiring result"))
|
|
persistent, _, err := service.materializeLocalBinaryResult(context.Background(), "task-expired", map[string]any{"b64_json": encoded})
|
|
if err != nil {
|
|
t.Fatalf("materialize fixture: %v", err)
|
|
}
|
|
entries, err := os.ReadDir(filepath.Join(service.localBinaryResultRoot(), "task-expired"))
|
|
if err != nil || len(entries) != 1 {
|
|
t.Fatalf("read fixture result: entries=%v err=%v", entries, err)
|
|
}
|
|
path := filepath.Join(service.localBinaryResultRoot(), "task-expired", entries[0].Name())
|
|
old := time.Now().Add(-2 * time.Hour)
|
|
if err := os.Chtimes(path, old, old); err != nil {
|
|
t.Fatalf("age fixture: %v", err)
|
|
}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
|
|
assertClientErrorCode(t, err, "binary_result_expired")
|
|
|
|
now := time.Now()
|
|
if err := os.Chtimes(path, now, now); err != nil {
|
|
t.Fatalf("refresh fixture: %v", err)
|
|
}
|
|
if err := os.WriteFile(path, []byte("tampered"), 0o640); err != nil {
|
|
t.Fatalf("tamper fixture: %v", err)
|
|
}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-expired", persistent)
|
|
assertClientErrorCode(t, err, "binary_result_corrupted")
|
|
}
|
|
|
|
func TestMaterializeLocalBinaryResultEnforcesLimitsAndKeepsText(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMaxBytes = 4
|
|
_, _, err := service.materializeLocalBinaryResult(context.Background(), "task-large", map[string]any{
|
|
"b64_json": base64.StdEncoding.EncodeToString([]byte("too large")),
|
|
})
|
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
|
|
|
persistent, changed, err := service.materializeLocalBinaryResult(context.Background(), "task-text", map[string]any{
|
|
"message": "YWJjZA==",
|
|
})
|
|
if err != nil || changed || persistent["message"] != "YWJjZA==" {
|
|
t.Fatalf("short Base64-like text should remain unchanged: result=%+v changed=%v err=%v", persistent, changed, err)
|
|
}
|
|
|
|
service = newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMaxBytes = 1024
|
|
service.cfg.LocalResultMaxTaskBytes = 8
|
|
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-total-large", map[string]any{
|
|
"first_base64": base64.StdEncoding.EncodeToString([]byte("12345")),
|
|
"second_base64": base64.StdEncoding.EncodeToString([]byte("67890")),
|
|
})
|
|
assertClientErrorCode(t, err, "binary_result_too_large")
|
|
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-total-large")); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("preflight task limit must not create partial files: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func TestCompactExpiredTaskResultDoesNotWriteFiles(t *testing.T) {
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMaxBytes = 1
|
|
encoded := base64.StdEncoding.EncodeToString([]byte("already expired result"))
|
|
|
|
persistent, changed, err := service.CompactExpiredTaskResultForStorage(context.Background(), "task-old", map[string]any{
|
|
"b64_json": encoded,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("compact expired result: %v", err)
|
|
}
|
|
if !changed || !localBinaryResultHasPlaceholders(persistent) {
|
|
t.Fatalf("expired result was not compacted: %+v", persistent)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-old")); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("expired compaction must not create a task directory: %v", err)
|
|
}
|
|
_, err = service.HydrateTaskResult(context.Background(), "task-old", persistent)
|
|
assertClientErrorCode(t, err, "binary_result_expired")
|
|
}
|
|
|
|
func TestLocalBinaryStorageUnavailableDoesNotRetryProvider(t *testing.T) {
|
|
err := localBinaryStorageError(errors.New("write failed"))
|
|
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
|
if clients.IsRetryable(err) {
|
|
t.Fatal("local result storage failure must not call the provider again")
|
|
}
|
|
if retryDecisionForCandidate(store.RuntimeModelCandidate{}, err).Retry {
|
|
t.Fatal("local result storage failure must not retry the same provider client")
|
|
}
|
|
if failoverDecisionForCandidate(store.RunnerPolicy{}, store.RuntimeModelCandidate{}, err).Retry {
|
|
t.Fatal("local result storage failure must not fail over to another provider")
|
|
}
|
|
|
|
service := newLocalBinaryTestService(t)
|
|
service.cfg.LocalResultMinFreeBytes = 1 << 62
|
|
_, _, err = service.materializeLocalBinaryResult(context.Background(), "task-low-disk", map[string]any{
|
|
"b64_json": base64.StdEncoding.EncodeToString([]byte("disk preflight")),
|
|
})
|
|
assertClientErrorCode(t, err, "local_result_storage_unavailable")
|
|
if _, statErr := os.Stat(filepath.Join(service.localBinaryResultRoot(), "task-low-disk")); !errors.Is(statErr, os.ErrNotExist) {
|
|
t.Fatalf("disk preflight must not create partial task files: %v", statErr)
|
|
}
|
|
}
|
|
|
|
func bytesToAny(payload []byte) []any {
|
|
result := make([]any, len(payload))
|
|
for index, value := range payload {
|
|
result[index] = float64(value)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func assertClientErrorCode(t *testing.T, err error, code string) {
|
|
t.Helper()
|
|
var clientErr *clients.ClientError
|
|
if !errors.As(err, &clientErr) || clientErr.Code != code {
|
|
t.Fatalf("error = %v, want client error code %s", err, code)
|
|
}
|
|
}
|