382 lines
12 KiB
Go
382 lines
12 KiB
Go
package runner
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"image"
|
|
imagedraw "image/draw"
|
|
"image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
_ "golang.org/x/image/bmp"
|
|
"golang.org/x/image/draw"
|
|
_ "golang.org/x/image/tiff"
|
|
_ "golang.org/x/image/webp"
|
|
)
|
|
|
|
const (
|
|
maxInputImageConversionBytes = 32 << 20
|
|
maxInputImageConversionPixels = 100_000_000
|
|
)
|
|
|
|
type inputImageResolutionRange struct {
|
|
MinLong int
|
|
MinShort int
|
|
MaxLong int
|
|
MaxShort int
|
|
}
|
|
|
|
type inputImageConstraints struct {
|
|
Resolution inputImageResolutionRange
|
|
MinAspect float64
|
|
MaxAspect float64
|
|
}
|
|
|
|
func (s *Service) normalizeVideoInputImages(ctx context.Context, body map[string]any, candidate store.RuntimeModelCandidate) (map[string]any, error) {
|
|
constraints, ok := candidateInputImageConstraints(candidate)
|
|
if !ok {
|
|
return body, nil
|
|
}
|
|
value, err := s.normalizeVideoInputImageValue(ctx, body, nil, constraints)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out, _ := value.(map[string]any)
|
|
if out == nil {
|
|
return map[string]any{}, nil
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *Service) normalizeVideoInputImageValue(ctx context.Context, value any, path []string, constraints inputImageConstraints) (any, error) {
|
|
switch typed := value.(type) {
|
|
case map[string]any:
|
|
next := make(map[string]any, len(typed))
|
|
for key, item := range typed {
|
|
normalized, err := s.normalizeVideoInputImageValue(ctx, item, append(path, key), constraints)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
next[key] = normalized
|
|
}
|
|
return next, nil
|
|
case []any:
|
|
next := make([]any, 0, len(typed))
|
|
for index, item := range typed {
|
|
normalized, err := s.normalizeVideoInputImageValue(ctx, item, append(path, fmt.Sprintf("[%d]", index)), constraints)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
next = append(next, normalized)
|
|
}
|
|
return next, nil
|
|
case string:
|
|
if !imageInputFieldNeedsHydration(path) {
|
|
return value, nil
|
|
}
|
|
normalized, converted, original, target, err := s.normalizeVideoInputImageSource(ctx, typed, constraints)
|
|
if err != nil {
|
|
param := requestInputImageParam(path)
|
|
return nil, &clients.ClientError{
|
|
Code: "invalid_parameter",
|
|
Message: fmt.Sprintf("输入图片 %s 自动转换失败：%s", param, err.Error()),
|
|
Param: param,
|
|
StatusCode: http.StatusBadRequest,
|
|
Retryable: false,
|
|
}
|
|
}
|
|
if converted && s.logger != nil {
|
|
s.logger.Info(
|
|
"video input image auto-converted",
|
|
"param", requestInputImageParam(path),
|
|
"original", fmt.Sprintf("%dx%d", original.X, original.Y),
|
|
"converted", fmt.Sprintf("%dx%d", target.X, target.Y),
|
|
)
|
|
}
|
|
return normalized, nil
|
|
default:
|
|
return value, nil
|
|
}
|
|
}
|
|
|
|
func candidateInputImageConstraints(candidate store.RuntimeModelCandidate) (inputImageConstraints, bool) {
|
|
modelTypes := []string{candidate.ModelType, "image_to_video", "omni_video"}
|
|
seen := map[string]bool{}
|
|
for _, modelType := range modelTypes {
|
|
modelType = strings.TrimSpace(modelType)
|
|
if modelType == "" || seen[modelType] {
|
|
continue
|
|
}
|
|
seen[modelType] = true
|
|
values, _ := candidate.Capabilities[modelType].(map[string]any)
|
|
if constraints, ok := parseInputImageConstraints(values); ok {
|
|
return constraints, true
|
|
}
|
|
}
|
|
return inputImageConstraints{}, false
|
|
}
|
|
|
|
func parseInputImageConstraints(values map[string]any) (inputImageConstraints, bool) {
|
|
if values == nil {
|
|
return inputImageConstraints{}, false
|
|
}
|
|
resolution, _ := values["input_image_resolution_range"].(map[string]any)
|
|
minimum, _ := resolution["min"].(map[string]any)
|
|
maximum, _ := resolution["max"].(map[string]any)
|
|
aspect, ok := inputImageNumberPair(values["input_image_aspect_ratio_range"])
|
|
constraints := inputImageConstraints{
|
|
Resolution: inputImageResolutionRange{
|
|
MinLong: int(math.Round(floatFromAny(minimum["long_edge"]))),
|
|
MinShort: int(math.Round(floatFromAny(minimum["short_edge"]))),
|
|
MaxLong: int(math.Round(floatFromAny(maximum["long_edge"]))),
|
|
MaxShort: int(math.Round(floatFromAny(maximum["short_edge"]))),
|
|
},
|
|
MinAspect: aspect[0],
|
|
MaxAspect: aspect[1],
|
|
}
|
|
if !ok ||
|
|
constraints.Resolution.MinLong <= 0 ||
|
|
constraints.Resolution.MinShort <= 0 ||
|
|
constraints.Resolution.MaxLong < constraints.Resolution.MinLong ||
|
|
constraints.Resolution.MaxShort < constraints.Resolution.MinShort ||
|
|
constraints.MinAspect <= 0 ||
|
|
constraints.MaxAspect < constraints.MinAspect {
|
|
return inputImageConstraints{}, false
|
|
}
|
|
return constraints, true
|
|
}
|
|
|
|
func inputImageNumberPair(value any) ([2]float64, bool) {
|
|
switch typed := value.(type) {
|
|
case []any:
|
|
if len(typed) != 2 {
|
|
return [2]float64{}, false
|
|
}
|
|
pair := [2]float64{floatFromAny(typed[0]), floatFromAny(typed[1])}
|
|
return pair, pair[0] > 0 && pair[1] > 0
|
|
case []float64:
|
|
if len(typed) != 2 {
|
|
return [2]float64{}, false
|
|
}
|
|
return [2]float64{typed[0], typed[1]}, typed[0] > 0 && typed[1] > 0
|
|
default:
|
|
return [2]float64{}, false
|
|
}
|
|
}
|
|
|
|
func (s *Service) normalizeVideoInputImageSource(ctx context.Context, source string, constraints inputImageConstraints) (string, bool, image.Point, image.Point, error) {
|
|
source = strings.TrimSpace(source)
|
|
if source == "" || strings.HasPrefix(strings.ToLower(source), "asset://") {
|
|
return source, false, image.Point{}, image.Point{}, nil
|
|
}
|
|
payload, err := s.readVideoInputImageBytes(ctx, source)
|
|
if err != nil {
|
|
return "", false, image.Point{}, image.Point{}, err
|
|
}
|
|
config, _, err := image.DecodeConfig(bytes.NewReader(payload))
|
|
if err != nil || config.Width <= 0 || config.Height <= 0 {
|
|
return "", false, image.Point{}, image.Point{}, fmt.Errorf("图片无法解码")
|
|
}
|
|
if config.Width > 50_000 || config.Height > 50_000 || int64(config.Width)*int64(config.Height) > maxInputImageConversionPixels {
|
|
return "", false, image.Point{}, image.Point{}, fmt.Errorf("图片像素数量超过转换上限")
|
|
}
|
|
original := image.Pt(config.Width, config.Height)
|
|
if inputImageWithinConstraints(original, constraints) {
|
|
return source, false, original, original, nil
|
|
}
|
|
|
|
decoded, _, err := image.Decode(bytes.NewReader(payload))
|
|
if err != nil {
|
|
return "", false, original, image.Point{}, fmt.Errorf("图片无法解码")
|
|
}
|
|
target := resolveInputImageTarget(original, constraints)
|
|
canvas := image.NewNRGBA(image.Rect(0, 0, target.X, target.Y))
|
|
imagedraw.Draw(canvas, canvas.Bounds(), &image.Uniform{C: averageImageColor(decoded)}, image.Point{}, imagedraw.Src)
|
|
|
|
sourceBounds := decoded.Bounds()
|
|
scale := math.Min(float64(target.X)/float64(sourceBounds.Dx()), float64(target.Y)/float64(sourceBounds.Dy()))
|
|
contentWidth := max(1, int(math.Round(float64(sourceBounds.Dx())*scale)))
|
|
contentHeight := max(1, int(math.Round(float64(sourceBounds.Dy())*scale)))
|
|
left := (target.X - contentWidth) / 2
|
|
top := (target.Y - contentHeight) / 2
|
|
draw.CatmullRom.Scale(
|
|
canvas,
|
|
image.Rect(left, top, left+contentWidth, top+contentHeight),
|
|
decoded,
|
|
sourceBounds,
|
|
draw.Over,
|
|
nil,
|
|
)
|
|
|
|
var output bytes.Buffer
|
|
if err := jpeg.Encode(&output, canvas, &jpeg.Options{Quality: 90}); err != nil {
|
|
return "", false, original, target, fmt.Errorf("图片编码失败")
|
|
}
|
|
if !inputImageWithinConstraints(target, constraints) {
|
|
return "", false, original, target, fmt.Errorf("转换结果仍不符合平台限制")
|
|
}
|
|
return "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString(output.Bytes()), true, original, target, nil
|
|
}
|
|
|
|
func (s *Service) readVideoInputImageBytes(ctx context.Context, source string) ([]byte, error) {
|
|
lower := strings.ToLower(source)
|
|
if strings.HasPrefix(lower, "data:") || (!strings.Contains(source, "://") && !strings.HasPrefix(source, "/")) {
|
|
payload, err := decodeBase64Payload(source)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("图片数据不是有效 Base64")
|
|
}
|
|
if len(payload) > maxInputImageConversionBytes {
|
|
return nil, fmt.Errorf("图片超过 32 MiB 转换上限")
|
|
}
|
|
return payload, nil
|
|
}
|
|
if localPath := s.localPathFromRequestAssetURL(source); localPath != "" {
|
|
payload, err := os.ReadFile(localPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("托管图片读取失败")
|
|
}
|
|
if len(payload) > maxInputImageConversionBytes {
|
|
return nil, fmt.Errorf("图片超过 32 MiB 转换上限")
|
|
}
|
|
return payload, nil
|
|
}
|
|
if !strings.HasPrefix(lower, "http://") && !strings.HasPrefix(lower, "https://") {
|
|
return nil, fmt.Errorf("图片地址格式不受支持")
|
|
}
|
|
|
|
requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, source, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("图片地址无效")
|
|
}
|
|
httpClient := generatedAssetHTTPClient(false)
|
|
httpClient.Timeout = 10 * time.Second
|
|
httpClient.CheckRedirect = func(request *http.Request, via []*http.Request) error {
|
|
if len(via) >= 3 {
|
|
return fmt.Errorf("图片地址重定向次数过多")
|
|
}
|
|
if !requestAssetURLIsPublic("", request.URL.String()) {
|
|
return fmt.Errorf("图片地址重定向到受限网络")
|
|
}
|
|
return nil
|
|
}
|
|
response, err := httpClient.Do(request)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("图片读取失败")
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("图片读取失败，HTTP %d", response.StatusCode)
|
|
}
|
|
payload, err := io.ReadAll(io.LimitReader(response.Body, maxInputImageConversionBytes+1))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("图片读取失败")
|
|
}
|
|
if len(payload) > maxInputImageConversionBytes {
|
|
return nil, fmt.Errorf("图片超过 32 MiB 转换上限")
|
|
}
|
|
return payload, nil
|
|
}
|
|
|
|
func inputImageWithinConstraints(size image.Point, constraints inputImageConstraints) bool {
|
|
longEdge := max(size.X, size.Y)
|
|
shortEdge := min(size.X, size.Y)
|
|
ratio := float64(size.X) / float64(size.Y)
|
|
return longEdge >= constraints.Resolution.MinLong &&
|
|
longEdge <= constraints.Resolution.MaxLong &&
|
|
shortEdge >= constraints.Resolution.MinShort &&
|
|
shortEdge <= constraints.Resolution.MaxShort &&
|
|
ratio >= constraints.MinAspect &&
|
|
ratio <= constraints.MaxAspect
|
|
}
|
|
|
|
func resolveInputImageTarget(source image.Point, constraints inputImageConstraints) image.Point {
|
|
sourceRatio := float64(source.X) / float64(source.Y)
|
|
targetRatio := math.Min(constraints.MaxAspect, math.Max(constraints.MinAspect, sourceRatio))
|
|
landscape := targetRatio >= 1
|
|
normalizedRatio := targetRatio
|
|
var sourceShort float64
|
|
if landscape {
|
|
sourceShort = math.Max(float64(source.Y), float64(source.X)/targetRatio)
|
|
} else {
|
|
normalizedRatio = 1 / targetRatio
|
|
sourceShort = math.Max(float64(source.X), float64(source.Y)*targetRatio)
|
|
}
|
|
|
|
shortEdge := int(math.Round(sourceShort))
|
|
shortEdge = min(constraints.Resolution.MaxShort, max(constraints.Resolution.MinShort, shortEdge))
|
|
longEdge := int(math.Floor(float64(shortEdge) * normalizedRatio))
|
|
if longEdge > constraints.Resolution.MaxLong {
|
|
longEdge = constraints.Resolution.MaxLong
|
|
shortEdge = int(math.Ceil(float64(longEdge) / normalizedRatio))
|
|
}
|
|
if longEdge < constraints.Resolution.MinLong {
|
|
longEdge = constraints.Resolution.MinLong
|
|
shortEdge = int(math.Ceil(float64(longEdge) / normalizedRatio))
|
|
}
|
|
shortEdge = min(constraints.Resolution.MaxShort, max(constraints.Resolution.MinShort, shortEdge))
|
|
longEdge = int(math.Floor(float64(shortEdge) * normalizedRatio))
|
|
longEdge = min(constraints.Resolution.MaxLong, max(constraints.Resolution.MinLong, longEdge))
|
|
if landscape {
|
|
return image.Pt(longEdge, shortEdge)
|
|
}
|
|
return image.Pt(shortEdge, longEdge)
|
|
}
|
|
|
|
func averageImageColor(source image.Image) imageColor {
|
|
bounds := source.Bounds()
|
|
stepX := max(1, bounds.Dx()/32)
|
|
stepY := max(1, bounds.Dy()/32)
|
|
var red, green, blue, count uint64
|
|
for y := bounds.Min.Y; y < bounds.Max.Y; y += stepY {
|
|
for x := bounds.Min.X; x < bounds.Max.X; x += stepX {
|
|
r, g, b, _ := source.At(x, y).RGBA()
|
|
red += uint64(r >> 8)
|
|
green += uint64(g >> 8)
|
|
blue += uint64(b >> 8)
|
|
count++
|
|
}
|
|
}
|
|
if count == 0 {
|
|
return imageColor{R: 255, G: 255, B: 255, A: 255}
|
|
}
|
|
return imageColor{R: uint8(red / count), G: uint8(green / count), B: uint8(blue / count), A: 255}
|
|
}
|
|
|
|
type imageColor struct {
|
|
R, G, B, A uint8
|
|
}
|
|
|
|
func (c imageColor) RGBA() (r, g, b, a uint32) {
|
|
return uint32(c.R) * 0x101, uint32(c.G) * 0x101, uint32(c.B) * 0x101, uint32(c.A) * 0x101
|
|
}
|
|
|
|
func requestInputImageParam(path []string) string {
|
|
var builder strings.Builder
|
|
for _, segment := range path {
|
|
if strings.HasPrefix(segment, "[") {
|
|
builder.WriteString(segment)
|
|
continue
|
|
}
|
|
if builder.Len() > 0 {
|
|
builder.WriteByte('.')
|
|
}
|
|
builder.WriteString(segment)
|
|
}
|
|
if builder.Len() == 0 {
|
|
return "image"
|
|
}
|
|
return builder.String()
|
|
}
|