From ddd68cfebdc2b06760d36796a0a5d32f2e10230b Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 21 Jul 2026 23:53:58 +0800 Subject: [PATCH] =?UTF-8?q?feat(seedance):=20=E6=8E=A5=E5=85=A5=E7=9C=9F?= =?UTF-8?q?=E4=BA=BA=E8=B5=84=E4=BA=A7=E7=AE=A1=E7=90=86=E4=B8=8E=E5=BC=95?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/api/internal/clients/volces_assets.go | 194 +++++++ .../internal/clients/volces_assets_test.go | 66 +++ .../httpapi/portrait_asset_handlers.go | 329 +++++++++++ apps/api/internal/runner/portrait_assets.go | 528 ++++++++++++++++++ .../internal/runner/portrait_assets_test.go | 50 ++ apps/api/internal/runner/service.go | 14 +- apps/api/internal/store/portrait_assets.go | 326 +++++++++++ .../0073_seedance_portrait_assets.sql | 78 +++ docs/seedance-portrait-assets.md | 43 ++ 9 files changed, 1627 insertions(+), 1 deletion(-) create mode 100644 apps/api/internal/clients/volces_assets.go create mode 100644 apps/api/internal/clients/volces_assets_test.go create mode 100644 apps/api/internal/httpapi/portrait_asset_handlers.go create mode 100644 apps/api/internal/runner/portrait_assets.go create mode 100644 apps/api/internal/runner/portrait_assets_test.go create mode 100644 apps/api/internal/store/portrait_assets.go create mode 100644 apps/api/migrations/0073_seedance_portrait_assets.sql create mode 100644 docs/seedance-portrait-assets.md diff --git a/apps/api/internal/clients/volces_assets.go b/apps/api/internal/clients/volces_assets.go new file mode 100644 index 0000000..3adb11b --- /dev/null +++ b/apps/api/internal/clients/volces_assets.go @@ -0,0 +1,194 @@ +package clients + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +const ( + volcesAssetDefaultEndpoint = "https://ark.cn-beijing.volcengineapi.com" + volcesAssetRegion = "cn-beijing" + volcesAssetService = "ark" + volcesAssetVersion = "2024-01-01" +) + +type VolcesAssetClient struct { + HTTPClient *http.Client + Now func() time.Time +} + +type VolcesAssetCredentials struct { + AccessKey string + SecretKey string + Endpoint string +} + +type VolcesAssetResult struct { + ID string `json:"Id"` + Name string `json:"Name,omitempty"` + URL string `json:"URL,omitempty"` + AssetType string `json:"AssetType,omitempty"` + GroupID string `json:"GroupId,omitempty"` + Status string `json:"Status,omitempty"` + Error map[string]any `json:"Error,omitempty"` + ProjectName string `json:"ProjectName,omitempty"` + CreateTime string `json:"CreateTime,omitempty"` + UpdateTime string `json:"UpdateTime,omitempty"` +} + +func (c VolcesAssetClient) CreateAsset(ctx context.Context, credentials VolcesAssetCredentials, body map[string]any) (VolcesAssetResult, string, error) { + var result struct { + ID string `json:"Id"` + } + requestID, err := c.call(ctx, credentials, "CreateAsset", body, &result) + return VolcesAssetResult{ID: result.ID}, requestID, err +} + +func (c VolcesAssetClient) GetAsset(ctx context.Context, credentials VolcesAssetCredentials, body map[string]any) (VolcesAssetResult, string, error) { + var result VolcesAssetResult + requestID, err := c.call(ctx, credentials, "GetAsset", body, &result) + return result, requestID, err +} + +func (c VolcesAssetClient) call(ctx context.Context, credentials VolcesAssetCredentials, action string, body map[string]any, target any) (string, error) { + accessKey := strings.TrimSpace(credentials.AccessKey) + secretKey := strings.TrimSpace(credentials.SecretKey) + if accessKey == "" || secretKey == "" { + return "", &ClientError{Code: "missing_credentials", Message: "volces portrait asset accessKey and secretKey are required", Retryable: false} + } + endpoint := strings.TrimRight(strings.TrimSpace(credentials.Endpoint), "/") + if endpoint == "" { + endpoint = volcesAssetDefaultEndpoint + } + baseURL, err := url.Parse(endpoint) + if err != nil || baseURL.Scheme == "" || baseURL.Host == "" { + return "", &ClientError{Code: "invalid_configuration", Message: "invalid volces portrait asset endpoint", Retryable: false} + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("marshal volces asset request: %w", err) + } + now := time.Now().UTC() + if c.Now != nil { + now = c.Now().UTC() + } + xDate := now.Format("20060102T150405Z") + contentSHA := sha256HexBytes(bodyJSON) + requestURL := *baseURL + requestURL.Path = "/" + requestURL.RawPath = "" + requestURL.RawQuery = canonicalVolcesAssetQuery(map[string]string{"Action": action, "Version": volcesAssetVersion}) + headers := map[string]string{ + "content-type": "application/json", + "host": baseURL.Host, + "x-content-sha256": contentSHA, + "x-date": xDate, + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), bytes.NewReader(bodyJSON)) + if err != nil { + return "", err + } + req.Host = baseURL.Host + req.Header.Set("Content-Type", headers["content-type"]) + req.Header.Set("X-Content-Sha256", headers["x-content-sha256"]) + req.Header.Set("X-Date", headers["x-date"]) + req.Header.Set("Authorization", volcesAssetAuthorization(accessKey, secretKey, http.MethodPost, "/", requestURL.RawQuery, headers, contentSHA, xDate)) + + response, err := httpClient(nil, c.HTTPClient).Do(req) + if err != nil { + return "", &ClientError{Code: "network", Message: err.Error(), Retryable: true} + } + defer response.Body.Close() + var envelope struct { + ResponseMetadata struct { + RequestID string `json:"RequestId"` + Error struct { + Code string `json:"Code"` + Message string `json:"Message"` + } `json:"Error"` + } `json:"ResponseMetadata"` + Result json.RawMessage `json:"Result"` + } + if err := json.NewDecoder(response.Body).Decode(&envelope); err != nil { + return requestIDFromHTTPResponse(response), &ClientError{Code: "invalid_response", Message: "decode volces portrait asset response: " + err.Error(), Retryable: HTTPRetryable(response.StatusCode), StatusCode: response.StatusCode} + } + requestID := firstNonEmpty(requestIDFromHTTPResponse(response), envelope.ResponseMetadata.RequestID) + if envelope.ResponseMetadata.Error.Code != "" || response.StatusCode >= http.StatusBadRequest { + message := strings.TrimSpace(envelope.ResponseMetadata.Error.Message) + if message == "" { + message = strings.TrimSpace(envelope.ResponseMetadata.Error.Code) + } + if message == "" { + message = fmt.Sprintf("volces %s failed with status %d", action, response.StatusCode) + } + return requestID, &ClientError{Code: firstNonEmpty(envelope.ResponseMetadata.Error.Code, "volces_asset_error"), Message: message, RequestID: requestID, StatusCode: response.StatusCode, Retryable: HTTPRetryable(response.StatusCode)} + } + if len(envelope.Result) == 0 || string(envelope.Result) == "null" { + return requestID, &ClientError{Code: "invalid_response", Message: "volces " + action + " returned empty result", RequestID: requestID, Retryable: false} + } + if err := json.Unmarshal(envelope.Result, target); err != nil { + return requestID, &ClientError{Code: "invalid_response", Message: "decode volces " + action + " result: " + err.Error(), RequestID: requestID, Retryable: false} + } + return requestID, nil +} + +func volcesAssetAuthorization(accessKey string, secretKey string, method string, path string, canonicalQuery string, headers map[string]string, bodySHA string, xDate string) string { + signedHeaders := []string{"content-type", "host", "x-content-sha256", "x-date"} + canonicalHeaderLines := make([]string, 0, len(signedHeaders)) + for _, key := range signedHeaders { + canonicalHeaderLines = append(canonicalHeaderLines, key+":"+strings.TrimSpace(headers[key])) + } + canonicalRequest := strings.Join([]string{ + strings.ToUpper(method), path, canonicalQuery, + strings.Join(canonicalHeaderLines, "\n") + "\n", + strings.Join(signedHeaders, ";"), bodySHA, + }, "\n") + date := xDate + if len(date) >= 8 { + date = date[:8] + } + scope := strings.Join([]string{date, volcesAssetRegion, volcesAssetService, "request"}, "/") + stringToSign := strings.Join([]string{"HMAC-SHA256", xDate, scope, sha256HexString(canonicalRequest)}, "\n") + kDate := hmacSHA256([]byte(secretKey), date) + kRegion := hmacSHA256(kDate, volcesAssetRegion) + kService := hmacSHA256(kRegion, volcesAssetService) + kSigning := hmacSHA256(kService, "request") + signature := hex.EncodeToString(hmacSHA256(kSigning, stringToSign)) + return "HMAC-SHA256 Credential=" + accessKey + "/" + scope + ", SignedHeaders=" + strings.Join(signedHeaders, ";") + ", Signature=" + signature +} + +func canonicalVolcesAssetQuery(values map[string]string) string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, url.QueryEscape(key)+"="+url.QueryEscape(values[key])) + } + return strings.ReplaceAll(strings.Join(parts, "&"), "+", "%20") +} + +func sha256HexBytes(value []byte) string { + digest := sha256.Sum256(value) + return hex.EncodeToString(digest[:]) +} + +func sha256HexString(value string) string { return sha256HexBytes([]byte(value)) } + +func hmacSHA256(key []byte, value string) []byte { + mac := hmac.New(sha256.New, key) + _, _ = mac.Write([]byte(value)) + return mac.Sum(nil) +} diff --git a/apps/api/internal/clients/volces_assets_test.go b/apps/api/internal/clients/volces_assets_test.go new file mode 100644 index 0000000..82684af --- /dev/null +++ b/apps/api/internal/clients/volces_assets_test.go @@ -0,0 +1,66 @@ +package clients + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestVolcesAssetClientSignsCreateAndReadsAsset(t *testing.T) { + var calls []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.URL.Query().Get("Action")) + if r.Method != http.MethodPost || r.URL.Path != "/" || r.URL.Query().Get("Version") != "2024-01-01" { + t.Fatalf("unexpected asset request %s %s?%s", r.Method, r.URL.Path, r.URL.RawQuery) + } + if r.Header.Get("X-Date") != "20260718T010203Z" { + t.Fatalf("unexpected x-date: %q", r.Header.Get("X-Date")) + } + if !strings.HasPrefix(r.Header.Get("Authorization"), "HMAC-SHA256 Credential=asset-ak/") { + t.Fatalf("missing Volces authorization: %q", r.Header.Get("Authorization")) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatalf("decode body: %v", err) + } + raw, _ := json.Marshal(body) + digest := sha256.Sum256(raw) + if got := r.Header.Get("X-Content-Sha256"); got != hex.EncodeToString(digest[:]) { + t.Fatalf("content hash mismatch got=%q", got) + } + switch r.URL.Query().Get("Action") { + case "CreateAsset": + if body["GroupId"] != "group-1" || body["AssetType"] != "Image" { + t.Fatalf("unexpected CreateAsset body: %+v", body) + } + _ = json.NewEncoder(w).Encode(map[string]any{"ResponseMetadata": map[string]any{"RequestId": "create-rid"}, "Result": map[string]any{"Id": "asset-1"}}) + case "GetAsset": + _ = json.NewEncoder(w).Encode(map[string]any{"ResponseMetadata": map[string]any{"RequestId": "get-rid"}, "Result": map[string]any{"Id": "asset-1", "Status": "Active", "AssetType": "Image"}}) + default: + t.Fatalf("unexpected Action: %q", r.URL.Query().Get("Action")) + } + })) + defer server.Close() + + client := VolcesAssetClient{HTTPClient: server.Client(), Now: func() time.Time { + return time.Date(2026, 7, 18, 1, 2, 3, 0, time.UTC) + }} + credentials := VolcesAssetCredentials{AccessKey: "asset-ak", SecretKey: "asset-sk", Endpoint: server.URL} + created, requestID, err := client.CreateAsset(context.Background(), credentials, map[string]any{"GroupId": "group-1", "URL": "https://example.com/person.png", "AssetType": "Image", "ProjectName": "default"}) + if err != nil || created.ID != "asset-1" || requestID != "create-rid" { + t.Fatalf("unexpected CreateAsset result=%+v requestID=%s err=%v", created, requestID, err) + } + asset, requestID, err := client.GetAsset(context.Background(), credentials, map[string]any{"Id": "asset-1", "ProjectName": "default"}) + if err != nil || asset.Status != "Active" || requestID != "get-rid" { + t.Fatalf("unexpected GetAsset result=%+v requestID=%s err=%v", asset, requestID, err) + } + if strings.Join(calls, ",") != "CreateAsset,GetAsset" { + t.Fatalf("unexpected actions: %+v", calls) + } +} diff --git a/apps/api/internal/httpapi/portrait_asset_handlers.go b/apps/api/internal/httpapi/portrait_asset_handlers.go new file mode 100644 index 0000000..898ada5 --- /dev/null +++ b/apps/api/internal/httpapi/portrait_asset_handlers.go @@ -0,0 +1,329 @@ +package httpapi + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/runner" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const seedancePortraitAssetCategory = "seedance_portrait_asset" + +// getSeedancePortraitAssetCapability godoc +// @Summary 查询 Seedance 真人资产能力 +// @Description 返回当前网关是否已配置可创建、同步和引用的火山 Seedance 真人资产平台。 +// @Tags portrait-assets +// @Produce json +// @Security BearerAuth +// @Success 200 {object} runner.PortraitAssetCapability +// @Router /api/v1/resource/material/seedance-portrait-assets/capability [get] +func (s *Server) getSeedancePortraitAssetCapability(w http.ResponseWriter, r *http.Request) { + capability, err := s.runner.PortraitAssetCapability(r.Context()) + if err != nil { + s.logger.Error("get portrait asset capability failed", "error", err) + writeError(w, http.StatusInternalServerError, "get portrait asset capability failed") + return + } + writeJSON(w, http.StatusOK, capability) +} + +// listSeedancePortraitAssets godoc +// @Summary 列出 Seedance 真人资产 +// @Description 返回当前用户的真人资产；兼容 server-main material 列表响应字段。 +// @Tags portrait-assets +// @Produce json +// @Security BearerAuth +// @Success 200 {object} map[string]any +// @Router /api/v1/resource/material/user/materials [get] +func (s *Server) listSeedancePortraitAssets(w http.ResponseWriter, r *http.Request) { + user, ok := auth.UserFromContext(r.Context()) + if !ok || user == nil { + writeError(w, http.StatusUnauthorized, "unauthorized") + return + } + if category := strings.TrimSpace(r.URL.Query().Get("category")); category != seedancePortraitAssetCategory { + writeError(w, http.StatusNotFound, "material category not found") + return + } + items, err := s.store.ListPortraitAssets(r.Context(), user, store.PortraitAssetListFilter{ + Keyword: r.URL.Query().Get("keyword"), + SourceType: firstNonEmptyQuery(r, "fileType", "sourceType"), + Page: portraitAssetQueryInt(r, "pageNumber", "page"), + PageSize: portraitAssetQueryInt(r, "pageSize"), + }) + if err != nil { + s.logger.Error("list portrait assets failed", "error", err) + writeError(w, http.StatusInternalServerError, "list portrait assets failed") + return + } + responseItems := make([]any, 0, len(items.Items)) + for _, item := range items.Items { + responseItems = append(responseItems, s.portraitAssetResponse(r, item)) + } + writeJSON(w, http.StatusOK, map[string]any{ + "data": responseItems, + "total": items.Total, + "page": items.Page, + "pageSize": items.PageSize, + }) +} + +// createSeedancePortraitAsset godoc +// @Summary 上传并创建 Seedance 真人资产 +// @Description 文件先写入网关文件存储；仅在 private_avatar_eligible=true 时登记到火山 Assets。创建后会立即触发一次状态同步。 +// @Tags portrait-assets +// @Accept multipart/form-data +// @Produce json +// @Security BearerAuth +// @Param file formData file true "真人资产源文件（图片、视频或音频）" +// @Param data formData string true "material JSON，category 必须是 seedance_portrait_asset" +// @Success 200 {object} map[string]any +// @Router /api/v1/resource/material [post] +func (s *Server) createSeedancePortraitAsset(w http.ResponseWriter, r *http.Request) { + user, ok := auth.UserFromContext(r.Context()) + if !ok || user == nil { + writeError(w, http.StatusUnauthorized, "unauthorized") + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxGatewayUploadBytes) + if err := r.ParseMultipartForm(multipartTaskMemoryBytes); err != nil { + writeError(w, http.StatusBadRequest, "invalid multipart form-data body") + return + } + if r.MultipartForm != nil { + defer r.MultipartForm.RemoveAll() + } + var data map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(r.FormValue("data"))), &data); err != nil || data == nil { + writeError(w, http.StatusBadRequest, "data must be a JSON object") + return + } + if strings.TrimSpace(portraitAssetString(data["category"])) != seedancePortraitAssetCategory { + writeError(w, http.StatusBadRequest, "category must be seedance_portrait_asset") + return + } + privateEligible, _ := data["private_avatar_eligible"].(bool) + if !privateEligible { + writeError(w, http.StatusBadRequest, "private_avatar_eligible must be true after the user confirms authorization", "portrait_asset_authorization_required") + return + } + file, header, err := r.FormFile("file") + if err != nil { + writeError(w, http.StatusBadRequest, "file is required") + return + } + defer file.Close() + payload, err := io.ReadAll(file) + if err != nil { + writeError(w, http.StatusBadRequest, "read portrait asset file failed") + return + } + contentType := strings.TrimSpace(header.Header.Get("Content-Type")) + if contentType == "" && len(payload) > 0 { + contentType = http.DetectContentType(payload) + } + sourceType := strings.ToLower(strings.TrimSpace(firstNonEmpty(portraitAssetString(data["fileType"]), portraitAssetString(data["sourceType"])))) + if !portraitAssetSourceMatchesContentType(sourceType, contentType) { + writeError(w, http.StatusBadRequest, "fileType must be image, video, or audio and match the uploaded file", "portrait_asset_unsupported_type") + return + } + upload, err := s.runner.UploadFile(r.Context(), runner.FileUploadPayload{ + Bytes: payload, ContentType: contentType, FileName: header.Filename, Source: "seedance-portrait-asset", Scene: store.FileStorageSceneUpload, + }) + if err != nil { + s.logger.Error("upload portrait asset failed", "error", err) + writeError(w, http.StatusBadGateway, err.Error(), clients.ErrorCode(err)) + return + } + url := strings.TrimSpace(portraitAssetString(upload["url"])) + if url == "" { + writeError(w, http.StatusBadGateway, "portrait asset upload returned no URL", "portrait_asset_source_url_required") + return + } + digest := sha256.Sum256(payload) + asset, reused, err := s.runner.CreatePortraitAsset(r.Context(), user, runner.PortraitAssetCreateInput{ + Name: strings.TrimSpace(portraitAssetString(data["name"])), + Description: strings.TrimSpace(portraitAssetString(data["description"])), + SourceType: sourceType, + URL: url, + Preview: firstNonEmpty(portraitAssetString(data["preview"]), url), + MimeType: contentType, + ByteSize: int64(len(payload)), + SourceSHA256: hex.EncodeToString(digest[:]), + PrivateAvatarEligible: privateEligible, + Metadata: map[string]any{ + "tags": data["tags"], + "materialGroupId": data["material_group_id"], + "uploadedFileName": header.Filename, + "uploadAssetStorage": upload["assetStorage"], + }, + }) + if err != nil { + writePortraitAssetError(w, err) + return + } + _, _ = s.runner.SyncPortraitAssets(r.Context(), user, []string{asset.ID}) + asset, _, err = s.refreshPortraitAssetForResponse(r, user, asset.ID, asset) + if err != nil { + s.logger.Error("refresh portrait asset after create failed", "error", err) + writeError(w, http.StatusInternalServerError, "refresh portrait asset failed") + return + } + response := map[string]any{"asset": s.portraitAssetResponse(r, asset)} + if reused { + response["dedupe"] = map[string]any{"reused": true, "code": "PORTRAIT_ASSET_REUSED", "reason": "same_source", "message": "已复用相同源文件的真人资产，并触发状态刷新。"} + } + writeJSON(w, http.StatusOK, response) +} + +// syncSeedancePortraitAssets godoc +// @Summary 同步 Seedance 真人资产状态 +// @Description 调用火山 CreateAsset/GetAsset；多次调用可把 Processing 状态刷新为 Active 或 Failed。 +// @Tags portrait-assets +// @Accept json +// @Produce json +// @Security BearerAuth +// @Success 200 {object} runner.PortraitAssetSyncResponse +// @Router /api/v1/resource/material/seedance-portrait-assets/sync [post] +func (s *Server) syncSeedancePortraitAssets(w http.ResponseWriter, r *http.Request) { + user, ok := auth.UserFromContext(r.Context()) + if !ok || user == nil { + writeError(w, http.StatusUnauthorized, "unauthorized") + return + } + var request struct { + IDs []string `json:"ids"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + writeError(w, http.StatusBadRequest, "invalid json body") + return + } + if len(request.IDs) == 0 { + writeJSON(w, http.StatusOK, runner.PortraitAssetSyncResponse{SyncedIDs: []string{}, Skipped: []runner.PortraitAssetIssue{}, Failed: []runner.PortraitAssetIssue{}, Assets: []store.PortraitAsset{}}) + return + } + response, err := s.runner.SyncPortraitAssets(r.Context(), user, request.IDs) + if err != nil { + s.logger.Error("sync portrait assets failed", "error", err) + writePortraitAssetError(w, err) + return + } + assets := make([]any, 0, len(response.Assets)) + for _, asset := range response.Assets { + assets = append(assets, s.portraitAssetResponse(r, asset)) + } + writeJSON(w, http.StatusOK, map[string]any{ + "requested": response.Requested, "accepted": response.Accepted, "syncedIds": response.SyncedIDs, + "skipped": response.Skipped, "failed": response.Failed, "assets": assets, + }) +} + +func (s *Server) refreshPortraitAssetForResponse(r *http.Request, user *auth.User, assetID string, fallback store.PortraitAsset) (store.PortraitAsset, bool, error) { + asset, found, err := s.store.FindPortraitAssetForUser(r.Context(), user, assetID) + if err != nil || !found { + return fallback, found, err + } + return asset, true, nil +} + +func (s *Server) portraitAssetResponse(r *http.Request, asset store.PortraitAsset) map[string]any { + active, total, lastError, updatedAt, err := s.store.PortraitAssetBindingSummary(r.Context(), asset.ID) + if err != nil { + active, total, lastError, updatedAt = 0, 0, asset.LastError, asset.UpdatedAt.UTC().Format(time.RFC3339Nano) + } + summaryStatus := asset.Status + if summaryStatus == "not_synced" && total == 0 { + summaryStatus = "not_synced" + } + response := map[string]any{ + "id": asset.ID, "name": asset.Name, "description": asset.Description, "url": asset.URL, "preview": firstNonEmpty(asset.Preview, asset.URL), + "type": "personal", "fileType": asset.SourceType, "sourceType": asset.SourceType, "size": asset.ByteSize, + "privateAvatarEligible": asset.PrivateAvatarEligible, + "createdAt": asset.CreatedAt.UTC().Format(time.RFC3339Nano), "updatedAt": asset.UpdatedAt.UTC().Format(time.RFC3339Nano), + "seedanceAssetSummary": map[string]any{ + "eligible": asset.PrivateAvatarEligible, "status": summaryStatus, "provider": "volces", "activePlatformCount": active, + "totalPlatformCount": total, "sourceType": asset.SourceType, "updatedAt": updatedAt, + }, + } + if lastError != "" { + response["seedanceAssetSummary"].(map[string]any)["lastError"] = lastError + } + if asset.SourceType == "image" { + response["thumbnail"] = firstNonEmpty(asset.Preview, asset.URL) + } + return response +} + +func writePortraitAssetError(w http.ResponseWriter, err error) { + status := http.StatusInternalServerError + if clientErr := clients.ErrorCode(err); clientErr != "client_error" { + switch clientErr { + case "portrait_asset_not_found": + status = http.StatusNotFound + case "portrait_asset_processing": + status = http.StatusServiceUnavailable + case "portrait_asset_authorization_required", "portrait_asset_unsupported_type", "portrait_asset_source_url_required", "portrait_asset_id_required", "portrait_asset_unsupported_model", "portrait_asset_audio_only": + status = http.StatusBadRequest + } + writeError(w, status, err.Error(), clientErr) + return + } + writeError(w, status, err.Error()) +} + +func portraitAssetSourceMatchesContentType(sourceType string, contentType string) bool { + contentType = strings.ToLower(strings.TrimSpace(contentType)) + switch sourceType { + case "image": + return strings.HasPrefix(contentType, "image/") + case "video": + return strings.HasPrefix(contentType, "video/") + case "audio": + return strings.HasPrefix(contentType, "audio/") + default: + return false + } +} + +func portraitAssetQueryInt(r *http.Request, keys ...string) int { + for _, key := range keys { + value := strings.TrimSpace(r.URL.Query().Get(key)) + if value == "" { + continue + } + var parsed int + if _, err := fmt.Sscan(value, &parsed); err == nil { + return parsed + } + } + return 0 +} + +func firstNonEmptyQuery(r *http.Request, keys ...string) string { + for _, key := range keys { + if value := strings.TrimSpace(r.URL.Query().Get(key)); value != "" { + return value + } + } + return "" +} + +func portraitAssetString(value any) string { + switch typed := value.(type) { + case string: + return typed + case fmt.Stringer: + return typed.String() + default: + return "" + } +} diff --git a/apps/api/internal/runner/portrait_assets.go b/apps/api/internal/runner/portrait_assets.go new file mode 100644 index 0000000..c082966 --- /dev/null +++ b/apps/api/internal/runner/portrait_assets.go @@ -0,0 +1,528 @@ +package runner + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +var portraitAssetPlaceholderPattern = regexp.MustCompile(`(?i)<<<[[:space:]]*portrait[_-]?asset_([0-9]+)[[:space:]]*>>>|@portrait_asset([0-9]+)|@人像资产([0-9]+)`) + +type PortraitAssetCapability struct { + Enabled bool `json:"enabled"` + CanUse bool `json:"canUse"` + CanCreate bool `json:"canCreate"` + CanUseAsPortraitAsset bool `json:"canUseAsPortraitAsset"` + CanUseAsPlainMaterial bool `json:"canUseAsPlainMaterial"` + AvailablePlatformIDs []string `json:"availablePlatformIds"` + CreationPlatformIDs []string `json:"creationPlatformIds"` + CanReferenceTencentAsset bool `json:"canReferenceTencentAssetUri"` + Reason string `json:"reason,omitempty"` +} + +type PortraitAssetCreateInput struct { + Name string + Description string + SourceType string + URL string + Preview string + MimeType string + ByteSize int64 + SourceSHA256 string + PrivateAvatarEligible bool + Metadata map[string]any +} + +type PortraitAssetSyncResponse struct { + Requested int `json:"requested"` + Accepted int `json:"accepted"` + SyncedIDs []string `json:"syncedIds"` + Skipped []PortraitAssetIssue `json:"skipped"` + Failed []PortraitAssetIssue `json:"failed"` + Assets []store.PortraitAsset `json:"assets"` +} + +type PortraitAssetIssue struct { + ID string `json:"id"` + Reason string `json:"reason"` +} + +type portraitAssetPlatformSettings struct { + ProjectName string + AssetGroupID string + Credentials clients.VolcesAssetCredentials +} + +func (s *Service) PortraitAssetCapability(ctx context.Context) (PortraitAssetCapability, error) { + platforms, err := s.store.ListPortraitAssetPlatforms(ctx) + if err != nil { + return PortraitAssetCapability{}, err + } + ids := make([]string, 0, len(platforms)) + for _, platform := range platforms { + if _, ok := portraitAssetSettings(platform); ok { + ids = append(ids, platform.PlatformID) + } + } + capability := PortraitAssetCapability{ + Enabled: len(ids) > 0, + CanUse: len(ids) > 0, + CanCreate: len(ids) > 0, + CanUseAsPortraitAsset: len(ids) > 0, + CanUseAsPlainMaterial: true, + AvailablePlatformIDs: ids, + CreationPlatformIDs: ids, + } + if len(ids) == 0 { + capability.Reason = "未配置可用的火山 Seedance 人像资产平台；请在 Volces 平台 config.seedancePrivateAsset 中配置 enabled、accessKey、secretKey、projectName、assetGroupId。" + } + return capability, nil +} + +func (s *Service) CreatePortraitAsset(ctx context.Context, user *auth.User, input PortraitAssetCreateInput) (store.PortraitAsset, bool, error) { + if s.store == nil { + return store.PortraitAsset{}, false, fmt.Errorf("portrait asset store is unavailable") + } + if !validPortraitAssetSourceType(input.SourceType) { + return store.PortraitAsset{}, false, &clients.ClientError{Code: "portrait_asset_unsupported_type", Message: "source type must be image, video, or audio", StatusCode: http.StatusBadRequest, Retryable: false} + } + if strings.TrimSpace(input.URL) == "" { + return store.PortraitAsset{}, false, &clients.ClientError{Code: "portrait_asset_source_url_required", Message: "portrait asset source URL is required", StatusCode: http.StatusBadRequest, Retryable: false} + } + if !input.PrivateAvatarEligible { + return store.PortraitAsset{}, false, &clients.ClientError{Code: "portrait_asset_authorization_required", Message: "private_avatar_eligible must be true after the user confirms authorization", StatusCode: http.StatusBadRequest, Retryable: false} + } + if existing, found, err := s.store.FindPortraitAssetBySourceHash(ctx, user, input.SourceSHA256); err != nil { + return store.PortraitAsset{}, false, err + } else if found { + return existing, true, nil + } + gatewayUserID, userID := portraitAssetUserKeys(user) + if user == nil || userID == "" { + return store.PortraitAsset{}, false, store.ErrLocalUserRequired + } + asset, err := s.store.CreatePortraitAsset(ctx, store.PortraitAssetInput{ + GatewayUserID: gatewayUserID, + UserID: userID, + GatewayTenantID: strings.TrimSpace(user.GatewayTenantID), + TenantID: strings.TrimSpace(user.TenantID), + TenantKey: strings.TrimSpace(user.TenantKey), + Name: strings.TrimSpace(input.Name), + Description: strings.TrimSpace(input.Description), + SourceType: strings.ToLower(strings.TrimSpace(input.SourceType)), + URL: strings.TrimSpace(input.URL), + Preview: firstNonEmptyString(strings.TrimSpace(input.Preview), strings.TrimSpace(input.URL)), + MimeType: strings.TrimSpace(input.MimeType), + ByteSize: input.ByteSize, + SourceSHA256: strings.TrimSpace(input.SourceSHA256), + PrivateAvatarEligible: input.PrivateAvatarEligible, + Metadata: input.Metadata, + }) + return asset, false, err +} + +func (s *Service) SyncPortraitAssets(ctx context.Context, user *auth.User, ids []string) (PortraitAssetSyncResponse, error) { + response := PortraitAssetSyncResponse{ + Requested: len(ids), SyncedIDs: make([]string, 0), Skipped: make([]PortraitAssetIssue, 0), Failed: make([]PortraitAssetIssue, 0), Assets: make([]store.PortraitAsset, 0), + } + platforms, err := s.store.ListPortraitAssetPlatforms(ctx) + if err != nil { + return response, err + } + configured := make([]store.PortraitAssetPlatform, 0, len(platforms)) + for _, platform := range platforms { + if _, ok := portraitAssetSettings(platform); ok { + configured = append(configured, platform) + } + } + seen := map[string]bool{} + for _, value := range ids { + assetID := strings.TrimSpace(value) + if assetID == "" || seen[assetID] { + continue + } + seen[assetID] = true + asset, found, err := s.store.FindPortraitAssetForUser(ctx, user, assetID) + if err != nil { + return response, err + } + if !found { + response.Skipped = append(response.Skipped, PortraitAssetIssue{ID: assetID, Reason: "portrait asset not found"}) + continue + } + if !asset.PrivateAvatarEligible { + response.Skipped = append(response.Skipped, PortraitAssetIssue{ID: asset.ID, Reason: "portrait asset authorization is required"}) + continue + } + if len(configured) == 0 { + _ = s.store.UpdatePortraitAssetStatus(ctx, asset.ID, "not_configured", "no configured Volces portrait asset platform") + asset.Status = "not_configured" + asset.LastError = "no configured Volces portrait asset platform" + response.Skipped = append(response.Skipped, PortraitAssetIssue{ID: asset.ID, Reason: asset.LastError}) + response.Assets = append(response.Assets, asset) + continue + } + + response.Accepted++ + assetFailed := false + for _, platform := range configured { + if err := s.syncPortraitAssetToPlatform(ctx, asset, platform); err != nil { + assetFailed = true + response.Failed = append(response.Failed, PortraitAssetIssue{ID: asset.ID, Reason: platform.PlatformID + ": " + err.Error()}) + } + } + updated, _, err := s.refreshPortraitAssetStatus(ctx, user, asset.ID) + if err != nil { + return response, err + } + response.Assets = append(response.Assets, updated) + if !assetFailed { + response.SyncedIDs = append(response.SyncedIDs, updated.ID) + } + } + return response, nil +} + +func (s *Service) syncPortraitAssetToPlatform(ctx context.Context, asset store.PortraitAsset, platform store.PortraitAssetPlatform) error { + settings, ok := portraitAssetSettings(platform) + if !ok { + return &clients.ClientError{Code: "portrait_asset_not_configured", Message: "platform portrait asset configuration is incomplete", Retryable: false} + } + binding, found, err := s.store.GetPortraitAssetBinding(ctx, asset.ID, platform.PlatformID) + if err != nil { + return err + } + if !found { + binding = store.PortraitAssetBinding{AssetID: asset.ID, PlatformID: platform.PlatformID, ProjectName: settings.ProjectName, AssetGroupID: settings.AssetGroupID, Status: "pending"} + } + if !portraitAssetHasPublicURL(asset.URL) { + return s.recordPortraitAssetBindingFailure(ctx, binding, settings, &clients.ClientError{ + Code: "portrait_asset_public_url_required", + Message: "portrait asset URL must be an absolute http(s) URL reachable by Volces", + StatusCode: http.StatusBadRequest, + Retryable: false, + }) + } + client := clients.VolcesAssetClient{HTTPClient: s.portraitAssetHTTPClient()} + remoteID := strings.TrimSpace(binding.RemoteAssetID) + if remoteID == "" { + created, _, createErr := client.CreateAsset(ctx, settings.Credentials, map[string]any{ + "GroupId": settings.AssetGroupID, "URL": asset.URL, "Name": asset.Name, + "AssetType": volcesPortraitAssetType(asset.SourceType), "ProjectName": settings.ProjectName, + }) + if createErr != nil { + return s.recordPortraitAssetBindingFailure(ctx, binding, settings, createErr) + } + remoteID = strings.TrimSpace(created.ID) + if remoteID == "" { + return s.recordPortraitAssetBindingFailure(ctx, binding, settings, &clients.ClientError{Code: "invalid_response", Message: "volces CreateAsset returned no asset id", Retryable: false}) + } + binding.RemoteAssetID = remoteID + } + remote, _, getErr := client.GetAsset(ctx, settings.Credentials, map[string]any{"Id": remoteID, "ProjectName": settings.ProjectName}) + if getErr != nil { + return s.recordPortraitAssetBindingFailure(ctx, binding, settings, getErr) + } + binding.ProjectName = settings.ProjectName + binding.AssetGroupID = settings.AssetGroupID + binding.RemoteAssetID = firstNonEmptyString(remote.ID, remoteID) + binding.RemoteAssetURI = "asset://" + binding.RemoteAssetID + binding.Status = portraitAssetBindingStatus(remote.Status) + binding.LastErrorCode = strings.TrimSpace(stringFromMap(remote.Error, "Code")) + binding.LastErrorMessage = strings.TrimSpace(stringFromMap(remote.Error, "Message")) + if binding.Status == "failed" && binding.LastErrorMessage == "" { + binding.LastErrorMessage = "volces portrait asset processing failed" + } + _, err = s.store.UpsertPortraitAssetBinding(ctx, binding) + return err +} + +func (s *Service) recordPortraitAssetBindingFailure(ctx context.Context, binding store.PortraitAssetBinding, settings portraitAssetPlatformSettings, cause error) error { + binding.ProjectName = settings.ProjectName + binding.AssetGroupID = settings.AssetGroupID + binding.Status = "failed" + binding.LastErrorCode = clients.ErrorCode(cause) + binding.LastErrorMessage = cause.Error() + _, err := s.store.UpsertPortraitAssetBinding(ctx, binding) + if err != nil { + return err + } + return cause +} + +func (s *Service) refreshPortraitAssetStatus(ctx context.Context, user *auth.User, assetID string) (store.PortraitAsset, bool, error) { + asset, found, err := s.store.FindPortraitAssetForUser(ctx, user, assetID) + if err != nil || !found { + return asset, found, err + } + active, total, lastError, _, err := s.store.PortraitAssetBindingSummary(ctx, asset.ID) + if err != nil { + return asset, true, err + } + status := "not_synced" + if total == 0 { + status = "not_synced" + } else if active > 0 { + status = "active" + if active < total { + status = "partial" + } + } else if lastError != "" { + status = "failed" + } else { + status = "pending" + } + if err := s.store.UpdatePortraitAssetStatus(ctx, asset.ID, status, lastError); err != nil { + return asset, true, err + } + asset.Status = status + asset.LastError = lastError + return asset, true, nil +} + +func (s *Service) compilePortraitAssetReferences(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) (map[string]any, error) { + entries := portraitAssetList(body["portrait_asset_list"]) + if len(entries) == 0 { + return body, nil + } + if kind != "videos.generations" || !isVolcesPortraitAssetCandidate(candidate) { + return nil, &clients.ClientError{Code: "portrait_asset_unsupported_model", Message: "portrait assets require a configured Volces Seedance omni video model", StatusCode: http.StatusBadRequest, Retryable: false} + } + if !candidateSupportsPortraitAssets(candidate) { + return nil, &clients.ClientError{Code: "portrait_asset_unsupported_model", Message: "selected model does not enable supports_portrait_asset_reference", StatusCode: http.StatusBadRequest, Retryable: false} + } + + out := cloneMap(body) + content := contentItems(out["content"]) + labels := make([]string, len(entries)) + nonAudioAssets := 0 + for index, entry := range entries { + assetID := firstNonEmptyString(stringFromMap(entry, "id"), stringFromMap(entry, "easyai_portrait_asset_id")) + if assetID == "" { + return nil, &clients.ClientError{Code: "portrait_asset_id_required", Message: fmt.Sprintf("portrait_asset_list[%d].id is required", index), StatusCode: http.StatusBadRequest, Retryable: false} + } + asset, found, err := s.store.FindPortraitAssetForUser(ctx, user, assetID) + if err != nil { + return nil, err + } + if !found { + return nil, &clients.ClientError{Code: "portrait_asset_not_found", Message: "portrait asset not found", StatusCode: http.StatusNotFound, Retryable: false} + } + if !asset.PrivateAvatarEligible { + return nil, &clients.ClientError{Code: "portrait_asset_authorization_required", Message: "portrait asset authorization is required", StatusCode: http.StatusBadRequest, Retryable: false} + } + binding, bound, err := s.store.GetPortraitAssetBinding(ctx, asset.ID, candidate.PlatformID) + if err != nil { + return nil, err + } + if !bound || binding.Status != "active" || strings.TrimSpace(binding.RemoteAssetURI) == "" { + return nil, &clients.ClientError{Code: "portrait_asset_processing", Message: "portrait asset is not active for the selected Volces platform; sync it and retry", StatusCode: http.StatusServiceUnavailable, Retryable: true} + } + labels[index] = firstNonEmptyString(strings.TrimSpace(stringFromMap(entry, "name")), asset.Name, "portrait asset "+fmt.Sprint(index+1)) + if asset.SourceType != "audio" { + nonAudioAssets++ + } + content = append(content, portraitAssetContent(asset.SourceType, binding.RemoteAssetURI)) + } + if nonAudioAssets == 0 { + return nil, &clients.ClientError{Code: "portrait_asset_audio_only", Message: "portrait_asset_list cannot contain audio-only assets", StatusCode: http.StatusBadRequest, Retryable: false} + } + for index := range content { + if strings.ToLower(strings.TrimSpace(stringFromAny(content[index]["type"]))) != "text" { + continue + } + content[index]["text"] = replacePortraitAssetPlaceholders(stringFromAny(content[index]["text"]), labels) + } + out["content"] = mapsToAnySlice(content) + delete(out, "portrait_asset_list") + return out, nil +} + +func (s *Service) portraitAssetHTTPClient() *http.Client { + if s.httpClients != nil && s.httpClients.none != nil { + return s.httpClients.none + } + return http.DefaultClient +} + +func portraitAssetSettings(platform store.PortraitAssetPlatform) (portraitAssetPlatformSettings, bool) { + config := portraitAssetNestedConfig(platform.Config) + accessKey := firstNonEmptyString(portraitAssetValue(config, "accessKey", "access_key"), portraitAssetValue(platform.Credentials, "accessKey", "access_key")) + secretKey := firstNonEmptyString(portraitAssetValue(config, "secretKey", "secret_key"), portraitAssetValue(platform.Credentials, "secretKey", "secret_key")) + projectName := firstNonEmptyString(portraitAssetValue(config, "projectName", "project_name"), "default") + assetGroupID := portraitAssetValue(config, "assetGroupId", "asset_group_id") + endpoint := firstNonEmptyString(portraitAssetValue(config, "assetEndpoint", "asset_endpoint", "volcesAssetEndpoint", "volces_asset_endpoint"), clientsVolcesAssetDefaultEndpoint()) + if accessKey == "" || secretKey == "" || projectName == "" || assetGroupID == "" { + return portraitAssetPlatformSettings{}, false + } + if enabled, present := portraitAssetBool(config, "enabled"); present && !enabled { + return portraitAssetPlatformSettings{}, false + } + return portraitAssetPlatformSettings{ProjectName: projectName, AssetGroupID: assetGroupID, Credentials: clients.VolcesAssetCredentials{AccessKey: accessKey, SecretKey: secretKey, Endpoint: endpoint}}, true +} + +func portraitAssetNestedConfig(config map[string]any) map[string]any { + for _, key := range []string{"seedancePrivateAsset", "seedance_private_asset", "portraitAsset", "portrait_asset"} { + if nested, ok := config[key].(map[string]any); ok { + return nested + } + } + return config +} + +func portraitAssetValue(values map[string]any, keys ...string) string { + for _, key := range keys { + if value := strings.TrimSpace(stringFromAny(values[key])); value != "" { + return value + } + } + return "" +} + +func portraitAssetBool(values map[string]any, key string) (bool, bool) { + value, ok := values[key] + if !ok { + return false, false + } + switch typed := value.(type) { + case bool: + return typed, true + case string: + return strings.EqualFold(strings.TrimSpace(typed), "true"), true + default: + return false, false + } +} + +func validPortraitAssetSourceType(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "image", "video", "audio": + return true + default: + return false + } +} + +func portraitAssetHasPublicURL(value string) bool { + parsed, err := url.Parse(strings.TrimSpace(value)) + if err != nil || parsed.Host == "" { + return false + } + return strings.EqualFold(parsed.Scheme, "http") || strings.EqualFold(parsed.Scheme, "https") +} + +func volcesPortraitAssetType(sourceType string) string { + switch strings.ToLower(strings.TrimSpace(sourceType)) { + case "video": + return "Video" + case "audio": + return "Audio" + default: + return "Image" + } +} + +func portraitAssetBindingStatus(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "active", "succeeded", "success": + return "active" + case "failed", "error": + return "failed" + default: + return "processing" + } +} + +func portraitAssetList(value any) []map[string]any { + switch typed := value.(type) { + case []any: + out := make([]map[string]any, 0, len(typed)) + for _, item := range typed { + if object, ok := item.(map[string]any); ok { + out = append(out, object) + } + } + return out + case []map[string]any: + return typed + default: + return nil + } +} + +func portraitAssetContent(sourceType string, assetURI string) map[string]any { + switch strings.ToLower(strings.TrimSpace(sourceType)) { + case "video": + return map[string]any{"type": "video_url", "role": "reference_video", "video_url": map[string]any{"url": assetURI}} + case "audio": + return map[string]any{"type": "audio_url", "role": "reference_audio", "audio_url": map[string]any{"url": assetURI}} + default: + return map[string]any{"type": "image_url", "role": "reference_image", "image_url": map[string]any{"url": assetURI}} + } +} + +func replacePortraitAssetPlaceholders(value string, labels []string) string { + return portraitAssetPlaceholderPattern.ReplaceAllStringFunc(value, func(match string) string { + parts := portraitAssetPlaceholderPattern.FindStringSubmatch(match) + for index := 1; index < len(parts); index++ { + if parts[index] == "" { + continue + } + position := int(parts[index][0] - '0') + if len(parts[index]) > 1 { + position = 0 + for _, r := range parts[index] { + position = position*10 + int(r-'0') + } + } + if position > 0 && position <= len(labels) && strings.TrimSpace(labels[position-1]) != "" { + return labels[position-1] + } + } + return match + }) +} + +func isVolcesPortraitAssetCandidate(candidate store.RuntimeModelCandidate) bool { + provider := strings.ToLower(strings.TrimSpace(candidate.Provider)) + return provider == "volces" || provider == "volces-openai" +} + +func candidateSupportsPortraitAssets(candidate store.RuntimeModelCandidate) bool { + capabilities := effectiveModelCapability(candidate) + for _, key := range []string{candidate.ModelType, "omni_video", "omni", "video_generate"} { + if capability, ok := capabilities[key].(map[string]any); ok { + if enabled, present := portraitAssetBool(capability, "supports_portrait_asset_reference"); present { + return enabled + } + } + } + return false +} + +func portraitAssetUserKeys(user *auth.User) (string, string) { + if user == nil { + return "", "" + } + gatewayUserID := strings.TrimSpace(user.GatewayUserID) + if gatewayUserID == "" && user.Source == "gateway" { + gatewayUserID = strings.TrimSpace(user.ID) + } + return gatewayUserID, strings.TrimSpace(user.ID) +} + +func portraitAssetSHA256(payload []byte) string { + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func clientsVolcesAssetDefaultEndpoint() string { return "https://ark.cn-beijing.volcengineapi.com" } diff --git a/apps/api/internal/runner/portrait_assets_test.go b/apps/api/internal/runner/portrait_assets_test.go new file mode 100644 index 0000000..8b81bfe --- /dev/null +++ b/apps/api/internal/runner/portrait_assets_test.go @@ -0,0 +1,50 @@ +package runner + +import ( + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestReplacePortraitAssetPlaceholders(t *testing.T) { + got := replacePortraitAssetPlaceholders("让 <<>> 和 @portrait_asset2、@人像资产3 出镜", []string{"Alice", "Bob", "Carol"}) + want := "让 Alice 和 Bob、Carol 出镜" + if got != want { + t.Fatalf("placeholder replacement = %q, want %q", got, want) + } +} + +func TestPortraitAssetContentUsesAssetURI(t *testing.T) { + item := portraitAssetContent("video", "asset://volces-video-1") + video, _ := item["video_url"].(map[string]any) + if item["type"] != "video_url" || item["role"] != "reference_video" || video["url"] != "asset://volces-video-1" { + t.Fatalf("unexpected portrait asset content: %+v", item) + } +} + +func TestPortraitAssetSettingsRequireConfiguredVolcesAssetGroup(t *testing.T) { + settings, ok := portraitAssetSettings(store.PortraitAssetPlatform{Config: map[string]any{ + "seedancePrivateAsset": map[string]any{ + "enabled": true, "accessKey": "ak", "secretKey": "sk", "projectName": "project", "assetGroupId": "group", + }, + }}) + if !ok || settings.ProjectName != "project" || settings.AssetGroupID != "group" || settings.Credentials.AccessKey != "ak" { + t.Fatalf("unexpected configured portrait asset settings: %+v ok=%v", settings, ok) + } + if _, ok := portraitAssetSettings(store.PortraitAssetPlatform{Config: map[string]any{"seedancePrivateAsset": map[string]any{"enabled": true, "accessKey": "ak"}}}); ok { + t.Fatal("incomplete platform config must not enable portrait assets") + } +} + +func TestPortraitAssetHasPublicURL(t *testing.T) { + for _, value := range []string{"https://assets.example.com/portrait.png", "http://assets.example.com/portrait.mp4"} { + if !portraitAssetHasPublicURL(value) { + t.Fatalf("expected public URL: %q", value) + } + } + for _, value := range []string{"/uploads/portrait.png", "file:///tmp/portrait.png", "asset://portrait-id"} { + if portraitAssetHasPublicURL(value) { + t.Fatalf("expected non-public URL: %q", value) + } + } +} diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 2f94a38..d6fafe4 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -916,7 +916,19 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user return clients.Response{}, fmt.Errorf("prepare http client: %w", err) } client := s.clientFor(candidate, simulated) - providerBody, err := s.hydrateProviderRequestAssets(ctx, body, candidate) + providerBody, err := s.compilePortraitAssetReferences(ctx, user, task.Kind, body, candidate) + if err != nil { + _ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{ + AttemptID: attemptID, + Status: "failed", + Retryable: false, + Metrics: mergeMetrics(baseAttemptMetrics, map[string]any{"error": err.Error(), "retryable": false, "trace": []any{failureTraceEntry(err, false)}}), + ErrorCode: clients.ErrorCode(err), + ErrorMessage: err.Error(), + }) + return clients.Response{}, err + } + providerBody, err = s.hydrateProviderRequestAssets(ctx, providerBody, candidate) if err != nil { _ = s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{ AttemptID: attemptID, diff --git a/apps/api/internal/store/portrait_assets.go b/apps/api/internal/store/portrait_assets.go new file mode 100644 index 0000000..3527351 --- /dev/null +++ b/apps/api/internal/store/portrait_assets.go @@ -0,0 +1,326 @@ +package store + +import ( + "context" + "encoding/json" + "strings" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" +) + +type PortraitAsset struct { + ID string `json:"id"` + GatewayUserID string `json:"gatewayUserId,omitempty"` + UserID string `json:"userId"` + GatewayTenantID string `json:"gatewayTenantId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + TenantKey string `json:"tenantKey,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + SourceType string `json:"sourceType"` + URL string `json:"url"` + Preview string `json:"preview,omitempty"` + MimeType string `json:"mimeType,omitempty"` + ByteSize int64 `json:"size,omitempty"` + SourceSHA256 string `json:"sourceSha256,omitempty"` + PrivateAvatarEligible bool `json:"privateAvatarEligible"` + Status string `json:"status"` + LastError string `json:"lastError,omitempty"` + Metadata map[string]any `json:"metadata,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type PortraitAssetBinding struct { + ID string `json:"id"` + AssetID string `json:"assetId"` + PlatformID string `json:"platformId"` + ProjectName string `json:"projectName,omitempty"` + AssetGroupID string `json:"assetGroupId,omitempty"` + RemoteAssetID string `json:"remoteAssetId,omitempty"` + RemoteAssetURI string `json:"remoteAssetUri,omitempty"` + Status string `json:"status"` + LastErrorCode string `json:"lastErrorCode,omitempty"` + LastErrorMessage string `json:"lastErrorMessage,omitempty"` + LastSyncedAt string `json:"lastSyncedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type PortraitAssetInput struct { + GatewayUserID string + UserID string + GatewayTenantID string + TenantID string + TenantKey string + Name string + Description string + SourceType string + URL string + Preview string + MimeType string + ByteSize int64 + SourceSHA256 string + PrivateAvatarEligible bool + Metadata map[string]any +} + +type PortraitAssetListFilter struct { + Keyword string + SourceType string + Page int + PageSize int +} + +type PortraitAssetListResult struct { + Items []PortraitAsset + Total int + Page int + PageSize int +} + +type PortraitAssetPlatform struct { + PlatformID string + PlatformKey string + Provider string + Credentials map[string]any + Config map[string]any +} + +const portraitAssetColumns = ` +a.id::text, COALESCE(a.gateway_user_id::text, ''), a.user_id, +COALESCE(a.gateway_tenant_id::text, ''), COALESCE(a.tenant_id, ''), COALESCE(a.tenant_key, ''), +a.name, a.description, a.source_type, a.url, a.preview, a.mime_type, a.byte_size, +a.source_sha256, a.private_avatar_eligible, a.status, a.last_error, a.metadata, a.created_at, a.updated_at` + +func (s *Store) CreatePortraitAsset(ctx context.Context, input PortraitAssetInput) (PortraitAsset, error) { + metadata, _ := json.Marshal(emptyObjectIfNil(input.Metadata)) + return scanPortraitAsset(s.pool.QueryRow(ctx, ` +INSERT INTO gateway_portrait_assets ( + gateway_user_id, user_id, gateway_tenant_id, tenant_id, tenant_key, + name, description, source_type, url, preview, mime_type, byte_size, source_sha256, + private_avatar_eligible, status, metadata +) +VALUES ( + NULLIF($1, '')::uuid, $2, NULLIF($3, '')::uuid, NULLIF($4, ''), NULLIF($5, ''), + $6, $7, $8, $9, $10, $11, $12, $13, $14, 'not_synced', $15::jsonb +) +RETURNING `+portraitAssetColumns, + input.GatewayUserID, input.UserID, input.GatewayTenantID, input.TenantID, input.TenantKey, + strings.TrimSpace(input.Name), strings.TrimSpace(input.Description), strings.TrimSpace(input.SourceType), + strings.TrimSpace(input.URL), strings.TrimSpace(input.Preview), strings.TrimSpace(input.MimeType), input.ByteSize, + strings.TrimSpace(input.SourceSHA256), input.PrivateAvatarEligible, string(metadata), + )) +} + +func (s *Store) FindPortraitAssetBySourceHash(ctx context.Context, user *auth.User, sourceSHA256 string) (PortraitAsset, bool, error) { + sourceSHA256 = strings.TrimSpace(sourceSHA256) + if sourceSHA256 == "" { + return PortraitAsset{}, false, nil + } + gatewayUserID, userID := portraitAssetUserKeys(user) + asset, err := scanPortraitAsset(s.pool.QueryRow(ctx, ` +SELECT `+portraitAssetColumns+` +FROM gateway_portrait_assets a +WHERE ((NULLIF($1, '')::uuid IS NOT NULL AND a.gateway_user_id = NULLIF($1, '')::uuid) + OR (NULLIF($2, '') IS NOT NULL AND a.user_id = $2)) + AND a.source_sha256 = $3 +ORDER BY a.created_at DESC +LIMIT 1`, gatewayUserID, userID, sourceSHA256)) + if IsNotFound(err) { + return PortraitAsset{}, false, nil + } + return asset, err == nil, err +} + +func (s *Store) FindPortraitAssetForUser(ctx context.Context, user *auth.User, assetID string) (PortraitAsset, bool, error) { + assetID = strings.TrimSpace(assetID) + if assetID == "" { + return PortraitAsset{}, false, nil + } + gatewayUserID, userID := portraitAssetUserKeys(user) + asset, err := scanPortraitAsset(s.pool.QueryRow(ctx, ` +SELECT `+portraitAssetColumns+` +FROM gateway_portrait_assets a +WHERE a.id = NULLIF($3, '')::uuid + AND ((NULLIF($1, '')::uuid IS NOT NULL AND a.gateway_user_id = NULLIF($1, '')::uuid) + OR (NULLIF($2, '') IS NOT NULL AND a.user_id = $2))`, gatewayUserID, userID, assetID)) + if IsNotFound(err) { + return PortraitAsset{}, false, nil + } + return asset, err == nil, err +} + +func (s *Store) ListPortraitAssets(ctx context.Context, user *auth.User, filter PortraitAssetListFilter) (PortraitAssetListResult, error) { + page := filter.Page + if page < 1 { + page = 1 + } + pageSize := filter.PageSize + if pageSize < 1 { + pageSize = 20 + } + if pageSize > 100 { + pageSize = 100 + } + gatewayUserID, userID := portraitAssetUserKeys(user) + keyword := strings.TrimSpace(filter.Keyword) + if keyword != "" { + keyword = "%" + keyword + "%" + } + where := ` +WHERE ((NULLIF($1, '')::uuid IS NOT NULL AND a.gateway_user_id = NULLIF($1, '')::uuid) + OR (NULLIF($2, '') IS NOT NULL AND a.user_id = $2)) + AND (NULLIF($3, '') IS NULL OR a.source_type = $3) + AND (NULLIF($4, '') IS NULL OR a.name ILIKE $4 OR a.description ILIKE $4)` + args := []any{gatewayUserID, userID, strings.TrimSpace(filter.SourceType), keyword} + var total int + if err := s.pool.QueryRow(ctx, `SELECT count(*) FROM gateway_portrait_assets a `+where, args...).Scan(&total); err != nil { + return PortraitAssetListResult{}, err + } + args = append(args, pageSize, (page-1)*pageSize) + rows, err := s.pool.Query(ctx, `SELECT `+portraitAssetColumns+` +FROM gateway_portrait_assets a `+where+` +ORDER BY a.created_at DESC +LIMIT $5 OFFSET $6`, args...) + if err != nil { + return PortraitAssetListResult{}, err + } + defer rows.Close() + items := make([]PortraitAsset, 0) + for rows.Next() { + asset, err := scanPortraitAsset(rows) + if err != nil { + return PortraitAssetListResult{}, err + } + items = append(items, asset) + } + if err := rows.Err(); err != nil { + return PortraitAssetListResult{}, err + } + return PortraitAssetListResult{Items: items, Total: total, Page: page, PageSize: pageSize}, nil +} + +func (s *Store) GetPortraitAssetBinding(ctx context.Context, assetID string, platformID string) (PortraitAssetBinding, bool, error) { + binding, err := scanPortraitAssetBinding(s.pool.QueryRow(ctx, ` +SELECT id::text, asset_id::text, platform_id::text, project_name, asset_group_id, + remote_asset_id, remote_asset_uri, status, last_error_code, last_error_message, + COALESCE(last_synced_at::text, ''), created_at, updated_at +FROM gateway_portrait_asset_bindings +WHERE asset_id = $1::uuid AND platform_id = $2::uuid`, assetID, platformID)) + if IsNotFound(err) { + return PortraitAssetBinding{}, false, nil + } + return binding, err == nil, err +} + +func (s *Store) UpsertPortraitAssetBinding(ctx context.Context, binding PortraitAssetBinding) (PortraitAssetBinding, error) { + return scanPortraitAssetBinding(s.pool.QueryRow(ctx, ` +INSERT INTO gateway_portrait_asset_bindings ( + asset_id, platform_id, project_name, asset_group_id, remote_asset_id, remote_asset_uri, + status, last_error_code, last_error_message, last_synced_at +) +VALUES ($1::uuid, $2::uuid, $3, $4, $5, $6, $7, $8, $9, now()) +ON CONFLICT (asset_id, platform_id) DO UPDATE SET + project_name = EXCLUDED.project_name, + asset_group_id = EXCLUDED.asset_group_id, + remote_asset_id = CASE WHEN EXCLUDED.remote_asset_id <> '' THEN EXCLUDED.remote_asset_id ELSE gateway_portrait_asset_bindings.remote_asset_id END, + remote_asset_uri = CASE WHEN EXCLUDED.remote_asset_uri <> '' THEN EXCLUDED.remote_asset_uri ELSE gateway_portrait_asset_bindings.remote_asset_uri END, + status = EXCLUDED.status, + last_error_code = EXCLUDED.last_error_code, + last_error_message = EXCLUDED.last_error_message, + last_synced_at = now(), + updated_at = now() +RETURNING id::text, asset_id::text, platform_id::text, project_name, asset_group_id, + remote_asset_id, remote_asset_uri, status, last_error_code, last_error_message, + COALESCE(last_synced_at::text, ''), created_at, updated_at`, + binding.AssetID, binding.PlatformID, strings.TrimSpace(binding.ProjectName), strings.TrimSpace(binding.AssetGroupID), + strings.TrimSpace(binding.RemoteAssetID), strings.TrimSpace(binding.RemoteAssetURI), strings.TrimSpace(binding.Status), + strings.TrimSpace(binding.LastErrorCode), strings.TrimSpace(binding.LastErrorMessage), + )) +} + +func (s *Store) UpdatePortraitAssetStatus(ctx context.Context, assetID string, status string, lastError string) error { + _, err := s.pool.Exec(ctx, ` +UPDATE gateway_portrait_assets +SET status = $2, last_error = $3, updated_at = now() +WHERE id = $1::uuid`, assetID, strings.TrimSpace(status), strings.TrimSpace(lastError)) + return err +} + +func (s *Store) PortraitAssetBindingSummary(ctx context.Context, assetID string) (active int, total int, latestError string, updatedAt string, err error) { + err = s.pool.QueryRow(ctx, ` +SELECT COUNT(*) FILTER (WHERE status = 'active'), COUNT(*), + COALESCE((ARRAY_AGG(NULLIF(last_error_message, '') ORDER BY updated_at DESC) FILTER (WHERE NULLIF(last_error_message, '') IS NOT NULL))[1], ''), + COALESCE(MAX(updated_at)::text, '') +FROM gateway_portrait_asset_bindings +WHERE asset_id = $1::uuid`, assetID).Scan(&active, &total, &latestError, &updatedAt) + return +} + +func (s *Store) ListPortraitAssetPlatforms(ctx context.Context) ([]PortraitAssetPlatform, error) { + rows, err := s.pool.Query(ctx, ` +SELECT p.id::text, p.platform_key, p.provider, p.credentials, p.config +FROM integration_platforms p +WHERE p.deleted_at IS NULL + AND p.status = 'enabled' + AND LOWER(p.provider) IN ('volces', 'volces-openai') +ORDER BY COALESCE(p.dynamic_priority, p.priority), p.created_at`) + if err != nil { + return nil, err + } + defer rows.Close() + items := make([]PortraitAssetPlatform, 0) + for rows.Next() { + var item PortraitAssetPlatform + var credentials, config []byte + if err := rows.Scan(&item.PlatformID, &item.PlatformKey, &item.Provider, &credentials, &config); err != nil { + return nil, err + } + item.Credentials = decodeObject(credentials) + item.Config = decodeObject(config) + items = append(items, item) + } + return items, rows.Err() +} + +func portraitAssetUserKeys(user *auth.User) (string, string) { + if user == nil { + return "", "" + } + gatewayUserID := strings.TrimSpace(user.GatewayUserID) + if gatewayUserID == "" && user.Source == "gateway" { + gatewayUserID = strings.TrimSpace(user.ID) + } + return gatewayUserID, strings.TrimSpace(user.ID) +} + +type portraitAssetScanner interface{ Scan(dest ...any) error } + +func scanPortraitAsset(scanner portraitAssetScanner) (PortraitAsset, error) { + var asset PortraitAsset + var metadata []byte + err := scanner.Scan( + &asset.ID, &asset.GatewayUserID, &asset.UserID, &asset.GatewayTenantID, &asset.TenantID, &asset.TenantKey, + &asset.Name, &asset.Description, &asset.SourceType, &asset.URL, &asset.Preview, &asset.MimeType, &asset.ByteSize, + &asset.SourceSHA256, &asset.PrivateAvatarEligible, &asset.Status, &asset.LastError, &metadata, &asset.CreatedAt, &asset.UpdatedAt, + ) + if err != nil { + return PortraitAsset{}, err + } + asset.Metadata = decodeObject(metadata) + return asset, nil +} + +func scanPortraitAssetBinding(scanner portraitAssetScanner) (PortraitAssetBinding, error) { + var binding PortraitAssetBinding + if err := scanner.Scan( + &binding.ID, &binding.AssetID, &binding.PlatformID, &binding.ProjectName, &binding.AssetGroupID, + &binding.RemoteAssetID, &binding.RemoteAssetURI, &binding.Status, &binding.LastErrorCode, &binding.LastErrorMessage, + &binding.LastSyncedAt, &binding.CreatedAt, &binding.UpdatedAt, + ); err != nil { + return PortraitAssetBinding{}, err + } + return binding, nil +} diff --git a/apps/api/migrations/0073_seedance_portrait_assets.sql b/apps/api/migrations/0073_seedance_portrait_assets.sql new file mode 100644 index 0000000..7e7d8ac --- /dev/null +++ b/apps/api/migrations/0073_seedance_portrait_assets.sql @@ -0,0 +1,78 @@ +CREATE TABLE IF NOT EXISTS gateway_portrait_assets ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + gateway_user_id uuid REFERENCES gateway_users(id) ON DELETE CASCADE, + user_id text NOT NULL, + gateway_tenant_id uuid REFERENCES gateway_tenants(id) ON DELETE SET NULL, + tenant_id text, + tenant_key text, + name text NOT NULL DEFAULT '', + description text NOT NULL DEFAULT '', + source_type text NOT NULL, + url text NOT NULL, + preview text NOT NULL DEFAULT '', + mime_type text NOT NULL DEFAULT '', + byte_size bigint NOT NULL DEFAULT 0, + source_sha256 text NOT NULL DEFAULT '', + private_avatar_eligible boolean NOT NULL DEFAULT false, + status text NOT NULL DEFAULT 'not_synced', + last_error text NOT NULL DEFAULT '', + metadata jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS gateway_portrait_asset_bindings ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + asset_id uuid NOT NULL REFERENCES gateway_portrait_assets(id) ON DELETE CASCADE, + platform_id uuid NOT NULL REFERENCES integration_platforms(id) ON DELETE CASCADE, + project_name text NOT NULL DEFAULT '', + asset_group_id text NOT NULL DEFAULT '', + remote_asset_id text NOT NULL DEFAULT '', + remote_asset_uri text NOT NULL DEFAULT '', + status text NOT NULL DEFAULT 'pending', + last_error_code text NOT NULL DEFAULT '', + last_error_message text NOT NULL DEFAULT '', + last_synced_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(asset_id, platform_id) +); + +CREATE INDEX IF NOT EXISTS idx_gateway_portrait_assets_user_created + ON gateway_portrait_assets(gateway_user_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_gateway_portrait_assets_user_id_created + ON gateway_portrait_assets(user_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_gateway_portrait_assets_user_hash + ON gateway_portrait_assets(gateway_user_id, source_sha256) + WHERE source_sha256 <> ''; + +CREATE INDEX IF NOT EXISTS idx_gateway_portrait_asset_bindings_asset_platform + ON gateway_portrait_asset_bindings(asset_id, platform_id); + +UPDATE base_model_catalog +SET capabilities = jsonb_set( + COALESCE(capabilities, '{}'::jsonb), + '{omni_video,supports_portrait_asset_reference}', + 'true'::jsonb, + true + ), + updated_at = now() +WHERE provider_key = 'volces' + AND provider_model_name LIKE 'doubao-seedance-2-0%' + AND model_type @> '["omni_video"]'::jsonb; + +UPDATE platform_models m +SET capabilities = jsonb_set( + COALESCE(m.capabilities, '{}'::jsonb), + '{omni_video,supports_portrait_asset_reference}', + 'true'::jsonb, + true + ), + updated_at = now() +FROM integration_platforms p +WHERE p.id = m.platform_id + AND p.provider = 'volces' + AND m.model_type @> '["omni_video"]'::jsonb + AND COALESCE(NULLIF(m.provider_model_name, ''), m.model_name) LIKE 'doubao-seedance-2-0%'; diff --git a/docs/seedance-portrait-assets.md b/docs/seedance-portrait-assets.md new file mode 100644 index 0000000..f046592 --- /dev/null +++ b/docs/seedance-portrait-assets.md @@ -0,0 +1,43 @@ +# Seedance 真人资产（Volces） + +网关会把真人源文件保存到既有文件存储，并在 Volces 平台的火山 Assets 库创建绑定。生成时只能引用已经对当前用户、当前平台激活的资产，最终发送给火山的视频内容 URL 为 `asset://`。 + +## 平台配置 + +在 `integration_platforms.config` 为对应的 `volces` 平台添加： + +```json +{ + "seedancePrivateAsset": { + "enabled": true, + "accessKey": "AK...", + "secretKey": "SK...", + "projectName": "default", + "assetGroupId": "asset-group-id", + "assetEndpoint": "https://ark.cn-beijing.volcengineapi.com" + } +} +``` + +`assetEndpoint` 可省略。源文件 URL 必须是火山可访问的绝对 `http(s)` 地址；同步时会拒绝本地路径、`file://` 与相对 URL。因此生产环境应配置带公网 URL 的文件存储或 `PublicBaseURL`。 + +## Desktop / server-main 兼容路由 + +- `GET /api/v1/resource/material/seedance-portrait-assets/capability` +- `GET /api/v1/resource/material/user/materials?category=seedance_portrait_asset` +- `POST /api/v1/resource/material`（multipart：`file`、`data`） +- `POST /api/v1/resource/material/seedance-portrait-assets/sync` +- `POST /api/v1/video/generations` 与 `GET /api/v1/ai/result/{taskID}` + +创建真人资产要求 `private_avatar_eligible: true`。同步接口可重复调用；资产在火山返回 `Active` 前，视频提交会返回 `portrait_asset_processing`，而不会把原始人像媒体当成普通参考图发送。 + +## 火山任务兼容路由 + +- `POST /api/v3/contents/generations/tasks` +- `GET /api/v3/contents/generations/tasks` +- `GET /api/v3/contents/generations/tasks/{taskID}` +- `DELETE /api/v3/contents/generations/tasks/{taskID}` + +列表接口兼容火山的 `page_num`、`page_size`、`filter.status`、`filter.task_ids`（可重复）和 `filter.model`，并返回官方 `items`、`total` 字段；`data`、`page` 是保留的网关附加字段。 + +公开 `id` 是网关任务 ID，原始火山任务 ID 保留在 `upstream_task_id`。响应保留火山的 `content`、`usage`、`seed`、`resolution` 等字段，并额外保留网关账单字段。