BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(storage): 统一二进制对象存储与公开错误

Browse Source 
新增 Aliyun OSS 与 S3 协议、通道内重试和按优先级跨通道切换，保留 server-main 兼容与环境 OSS 内存通道。

将请求及结果中的 Base64、Data URI、Buffer、multipart 和内联二进制统一对象化，生产路径不再写入本机静态目录，历史本地资源仅保留只读兼容。

引入 PublicErrorV1 并统一 API、异步查询、兼容协议和失败回调的安全错误输出，同时补充迁移、管理端、指标、OpenAPI 与本地模拟验收。

验证：go test ./... -count=1；go vet ./...；pnpm lint；pnpm test；pnpm build；pnpm openapi；tests/ci/migrations-test.sh。
This commit is contained in:
wangbo
2026-08-04 08:14:39 +08:00
parent d129bcccbd
commit 0f0998cbcf
55 changed files with 3649 additions and 1008 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/store/file_storage_channels.go
+77 -20
View File
@@ -39,6 +39,9 @@ type FileStorageChannel struct {
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl,omitempty"`
APIKey string `json:"-"`
AccessKeyID string `json:"-"`
AccessKeySecret string `json:"-"`
SessionToken string `json:"-"`
CredentialsPreview map[string]any `json:"credentialsPreview,omitempty"`
Scenes []string `json:"scenes,omitempty"`
Config map[string]any `json:"config,omitempty"`
@@ -53,16 +56,21 @@ type FileStorageChannel struct {
}
type FileStorageChannelInput struct {
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl"`
APIKey *string `json:"apiKey"`
Scenes []string `json:"scenes"`
Config map[string]any `json:"config"`
RetryPolicy map[string]any `json:"retryPolicy"`
Priority int `json:"priority"`
Status string `json:"status"`
ChannelKey string `json:"channelKey"`
Name string `json:"name"`
Provider string `json:"provider"`
UploadURL string `json:"uploadUrl"`
APIKey *string `json:"apiKey"`
AccessKey *string `json:"accessKey"`
AccessKeyID *string `json:"accessKeyId"`
AccessKeySecret *string `json:"accessKeySecret"`
SecretKey *string `json:"secretKey"`
SessionToken *string `json:"sessionToken"`
Scenes []string `json:"scenes"`
Config map[string]any `json:"config"`
RetryPolicy map[string]any `json:"retryPolicy"`
Priority int `json:"priority"`
Status string `json:"status"`
}
type FileStorageSettings struct {
@@ -156,11 +164,18 @@ WHERE id = $1::uuid
AND deleted_at IS NULL`, id))
}
func (s *Store) GetFileStorageChannelByKey(ctx context.Context, channelKey string) (FileStorageChannel, error) {
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
SELECT `+fileStorageChannelColumns+`
FROM file_storage_channels
WHERE channel_key = $1`, strings.TrimSpace(channelKey)))
}
func (s *Store) CreateFileStorageChannel(ctx context.Context, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
INSERT INTO file_storage_channels (
@@ -182,10 +197,10 @@ RETURNING `+fileStorageChannelColumns,
func (s *Store) UpdateFileStorageChannel(ctx context.Context, id string, input FileStorageChannelInput) (FileStorageChannel, error) {
input = normalizeFileStorageChannelInput(input)
replaceCredentials := input.APIKey != nil
replaceCredentials := fileStorageInputReplacesCredentials(input)
credentials, _ := json.Marshal(credentialsFromFileStorageInput(input))
config, _ := json.Marshal(configFromFileStorageInput(input))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy))
retryPolicy, _ := json.Marshal(defaultFileStorageRetryPolicyIfEmpty(input.RetryPolicy, input.Provider))
return scanFileStorageChannel(s.pool.QueryRow(ctx, `
UPDATE file_storage_channels
@@ -193,7 +208,7 @@ SET channel_key = $2,
name = $3,
provider = $4,
upload_url = NULLIF($5, ''),
credentials = CASE WHEN $6::boolean THEN $7 ELSE credentials END,
credentials = CASE WHEN $6::boolean THEN credentials || $7::jsonb ELSE credentials END,
config = $8,
retry_policy = $9,
priority = $10,
@@ -307,6 +322,9 @@ func scanFileStorageChannel(scanner fileStorageChannelScanner) (FileStorageChann
}
credentialObject := decodeObject(credentials)
item.APIKey = stringFromObject(credentialObject, "apiKey")
item.AccessKeyID = stringFromObject(credentialObject, "accessKeyId")
item.AccessKeySecret = stringFromObject(credentialObject, "accessKeySecret")
item.SessionToken = stringFromObject(credentialObject, "sessionToken")
item.CredentialsPreview = maskCredentialsPreview(credentials)
configObject := decodeObject(config)
item.Scenes = fileStorageScenesFromConfig(configObject)
@@ -324,6 +342,17 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
apiKey := strings.TrimSpace(*input.APIKey)
input.APIKey = &apiKey
}
if input.AccessKeyID == nil && input.AccessKey != nil {
input.AccessKeyID = input.AccessKey
}
if input.AccessKeySecret == nil && input.SecretKey != nil {
input.AccessKeySecret = input.SecretKey
}
for _, value := range []*string{input.AccessKeyID, input.AccessKeySecret, input.SessionToken} {
if value != nil {
*value = strings.TrimSpace(*value)
}
}
input.Scenes = normalizeFileStorageScenes(input.Scenes)
input.Status = strings.ToLower(strings.TrimSpace(input.Status))
if input.Provider == "" {
@@ -342,11 +371,24 @@ func normalizeFileStorageChannelInput(input FileStorageChannelInput) FileStorage
}
func credentialsFromFileStorageInput(input FileStorageChannelInput) map[string]any {
apiKey := fileStorageInputAPIKey(input)
if apiKey == "" {
return map[string]any{}
credentials := map[string]any{}
if input.APIKey != nil {
credentials["apiKey"] = fileStorageInputAPIKey(input)
}
return map[string]any{"apiKey": apiKey}
if input.AccessKeyID != nil {
credentials["accessKeyId"] = strings.TrimSpace(*input.AccessKeyID)
}
if input.AccessKeySecret != nil {
credentials["accessKeySecret"] = strings.TrimSpace(*input.AccessKeySecret)
}
if input.SessionToken != nil {
credentials["sessionToken"] = strings.TrimSpace(*input.SessionToken)
}
return credentials
}
func fileStorageInputReplacesCredentials(input FileStorageChannelInput) bool {
return input.APIKey != nil || input.AccessKeyID != nil || input.AccessKeySecret != nil || input.AccessKey != nil || input.SecretKey != nil || input.SessionToken != nil
}
func fileStorageInputAPIKey(input FileStorageChannelInput) string {
@@ -549,7 +591,10 @@ func NormalizeFileStorageResultUploadPolicy(policy string) string {
case "upload_all", "all", "always", "all_upload":
return FileStorageResultUploadPolicyUploadAll
case "upload_none", "none", "never", "disabled", "no_upload", "skip", "skip_all":
return FileStorageResultUploadPolicyUploadNone
// Local result persistence is no longer a supported write path. Preserve
// compatibility with historical settings by normalizing them to the safe
// object-storage policy.
return FileStorageResultUploadPolicyDefault
default:
return FileStorageResultUploadPolicyDefault
}
@@ -601,10 +646,22 @@ func defaultFileStorageScenes() []string {
return []string{FileStorageSceneUpload, FileStorageSceneImageResult, FileStorageSceneRequestAsset}
}
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any) map[string]any {
func defaultFileStorageRetryPolicyIfEmpty(policy map[string]any, providers ...string) map[string]any {
if len(policy) > 0 {
return policy
}
provider := ""
if len(providers) > 0 {
provider = strings.ToLower(strings.TrimSpace(providers[0]))
}
if provider == "aliyun_oss" || provider == "s3" {
return map[string]any{
"enabled": true,
"maxRetries": 2,
"backoffSeconds": []any{0.25, 1.0},
"strategy": "exponential",
}
}
return map[string]any{
"enabled": true,
"maxRetries": 3,
apps/api/internal/store/file_storage_channels_test.go
+31
View File
@@ -1,7 +1,9 @@
package store
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
@@ -15,3 +17,32 @@ func TestDefaultFileStorageScenesIncludeCrossNodeRequestAssets(t *testing.T) {
t.Fatalf("default file storage scenes = %#v, want %#v", got, want)
}
}
func TestFileStorageChannelJSONNeverIncludesCredentials(t *testing.T) {
channel := FileStorageChannel{
Provider: "s3", APIKey: "api-key-secret", AccessKeyID: "access-key-id",
AccessKeySecret: "access-key-secret", SessionToken: "session-token",
CredentialsPreview: map[string]any{"accessKeyId": "acce…-id", "accessKeySecret": "acce…cret"},
}
payload, err := json.Marshal(channel)
if err != nil {
t.Fatal(err)
}
encoded := string(payload)
for _, secret := range []string{"api-key-secret", "access-key-id", "access-key-secret", "session-token"} {
if strings.Contains(encoded, secret) {
t.Fatalf("channel JSON leaked credential %q: %s", secret, encoded)
}
}
if !strings.Contains(encoded, "credentialsPreview") {
t.Fatalf("channel JSON omitted masked preview: %s", encoded)
}
}
func TestLegacyUploadNoneNormalizesToObjectStorageDefault(t *testing.T) {
for _, value := range []string{"upload_none", "none", "never", "skip_all"} {
if got := NormalizeFileStorageResultUploadPolicy(value); got != FileStorageResultUploadPolicyDefault {
t.Fatalf("policy %q normalized to %q", value, got)
}
}
}
apps/api/internal/store/postgres.go
+122 -101
View File
@@ -14,6 +14,7 @@ import (
"unicode"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
@@ -568,73 +569,74 @@ type CreateTaskResult struct {
}
type GatewayTask struct {
ID string `json:"id"`
ExternalTaskID string `json:"externalTaskId,omitempty"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
AcceptanceRunID string `json:"acceptanceRunId,omitempty"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
QueueKey string `json:"-"`
Priority int `json:"-"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"-"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
BillingVersion string `json:"billingVersion"`
BillingStatus string `json:"billingStatus"`
BillingCurrency string `json:"billingCurrency"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
ReservationAmount float64 `json:"reservationAmount"`
ExecutionToken string `json:"-"`
ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"`
CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"`
CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
ExternalTaskID string `json:"externalTaskId,omitempty"`
Kind string `json:"kind"`
RunMode string `json:"runMode"`
AcceptanceRunID string `json:"acceptanceRunId,omitempty"`
UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"`
ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"`
QueueKey string `json:"-"`
Priority int `json:"-"`
Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"-"`
Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"`
BillingVersion string `json:"billingVersion"`
BillingStatus string `json:"billingStatus"`
BillingCurrency string `json:"billingCurrency"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
ReservationAmount float64 `json:"reservationAmount"`
ExecutionToken string `json:"-"`
ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
PublicError *publicerror.Error `json:"publicError,omitempty"`
CompatibilityProtocol string `json:"compatibilityProtocol,omitempty"`
CompatibilityPublicID string `json:"compatibilityPublicId,omitempty"`
CompatibilitySourceProtocol string `json:"compatibilitySourceProtocol,omitempty"`
CompatibilitySubmitHTTPStatus int `json:"compatibilitySubmitHttpStatus,omitempty"`
CompatibilitySubmitHeaders map[string]any `json:"compatibilitySubmitHeaders,omitempty"`
CompatibilitySubmitBody map[string]any `json:"compatibilitySubmitBody,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
const gatewayTaskColumns = `
@@ -656,7 +658,7 @@ COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''),
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''), COALESCE(public_error, '{}'::jsonb),
COALESCE(compatibility_protocol, ''), COALESCE(compatibility_public_id, ''),
COALESCE(compatibility_source_protocol, ''), COALESCE(compatibility_submit_http_status, 0),
COALESCE(compatibility_submit_headers, '{}'::jsonb), COALESCE(compatibility_submit_body, '{}'::jsonb),
@@ -679,39 +681,40 @@ type TaskEvent struct {
}
type TaskAttempt struct {
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
ID string `json:"id"`
TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"`
Status string `json:"status"`
Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
RequestFingerprint string `json:"requestFingerprint,omitempty"`
UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
PublicError *publicerror.Error `json:"publicError,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
}
type TaskParamPreprocessingLog struct {
@@ -2240,6 +2243,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
var remoteTaskPayloadBytes []byte
var compatibilitySubmitHeadersBytes []byte
var compatibilitySubmitBodyBytes []byte
var publicErrorBytes []byte
if err := scanner.Scan(
&task.ID,
&task.ExternalTaskID,
@@ -2295,6 +2299,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
&task.Error,
&task.ErrorCode,
&task.ErrorMessage,
&publicErrorBytes,
&task.CompatibilityProtocol,
&task.CompatibilityPublicID,
&task.CompatibilitySourceProtocol,
@@ -2317,9 +2322,25 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
task.CompatibilitySubmitHeaders = decodeObject(compatibilitySubmitHeadersBytes)
task.CompatibilitySubmitBody = decodeObject(compatibilitySubmitBodyBytes)
if len(publicErrorBytes) > 0 {
var snapshot publicerror.Error
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
task.PublicError = &snapshot
}
}
return task, nil
}
func encodePublicErrorSnapshot(code string, message string, status int, retryable bool, requestID string, taskID string) []byte {
if strings.TrimSpace(code) == "" && strings.TrimSpace(message) == "" {
return []byte("null")
}
snapshot := publicerror.FromFields(code, message, status, retryable)
snapshot = publicerror.WithIDs(snapshot, requestID, taskID)
payload, _ := json.Marshal(snapshot)
return payload
}
func (s *Store) ListTaskEvents(ctx context.Context, taskID string) ([]TaskEvent, error) {
rows, err := s.pool.Query(ctx, `
SELECT id::text, task_id::text, seq, event_type, COALESCE(status, ''), COALESCE(phase, ''),
apps/api/internal/store/request_assets.go
+47 -21
View File
@@ -9,33 +9,43 @@ import (
)
type RequestAsset struct {
ID string `json:"id"`
SHA256 string `json:"sha256"`
ContentType string `json:"contentType"`
ByteSize int64 `json:"byteSize"`
URL string `json:"url"`
StorageProvider string `json:"storageProvider"`
LocalPath string `json:"localPath,omitempty"`
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
RefCount int `json:"refCount"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID string `json:"id"`
SHA256 string `json:"sha256"`
ContentType string `json:"contentType"`
ByteSize int64 `json:"byteSize"`
URL string `json:"url"`
StorageProvider string `json:"storageProvider"`
StorageChannelID string `json:"storageChannelId,omitempty"`
StorageChannelKey string `json:"storageChannelKey,omitempty"`
ObjectKey string `json:"objectKey,omitempty"`
AccessScope string `json:"accessScope,omitempty"`
LocalPath string `json:"localPath,omitempty"`
ExpiresAt *time.Time `json:"expiresAt,omitempty"`
ExpiredAt *time.Time `json:"expiredAt,omitempty"`
RefCount int `json:"refCount"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type RequestAssetInput struct {
SHA256 string
ContentType string
ByteSize int64
URL string
StorageProvider string
LocalPath string
ExpiresAt *time.Time
SHA256 string
ContentType string
ByteSize int64
URL string
StorageProvider string
StorageChannelID string
StorageChannelKey string
ObjectKey string
AccessScope string
LocalPath string
ExpiresAt *time.Time
}
func (s *Store) FindRequestAsset(ctx context.Context, sha256 string, contentType string) (RequestAsset, bool, error) {
asset, err := scanRequestAsset(s.pool.QueryRow(ctx, `
SELECT id::text, sha256, content_type, byte_size, url, storage_provider,
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at
FROM gateway_request_assets
WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
@@ -51,25 +61,37 @@ WHERE sha256 = $1 AND content_type = $2`, sha256, contentType))
func (s *Store) UpsertRequestAsset(ctx context.Context, input RequestAssetInput) (RequestAsset, error) {
return scanRequestAsset(s.pool.QueryRow(ctx, `
INSERT INTO gateway_request_assets (
sha256, content_type, byte_size, url, storage_provider, local_path, expires_at, expired_at, ref_count
sha256, content_type, byte_size, url, storage_provider, storage_channel_id,
storage_channel_key, object_key, access_scope, local_path, expires_at, expired_at, ref_count
)
VALUES ($1, $2, $3, $4, $5, NULLIF($6, ''), $7, NULL, 1)
VALUES ($1, $2, $3, $4, $5, NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''),
COALESCE(NULLIF($9, ''), 'private'), NULLIF($10, ''), $11, NULL, 1)
ON CONFLICT (sha256, content_type) DO UPDATE
SET byte_size = EXCLUDED.byte_size,
url = EXCLUDED.url,
storage_provider = EXCLUDED.storage_provider,
storage_channel_id = EXCLUDED.storage_channel_id,
storage_channel_key = EXCLUDED.storage_channel_key,
object_key = EXCLUDED.object_key,
access_scope = EXCLUDED.access_scope,
local_path = EXCLUDED.local_path,
expires_at = EXCLUDED.expires_at,
expired_at = NULL,
ref_count = gateway_request_assets.ref_count + 1,
updated_at = now()
RETURNING id::text, sha256, content_type, byte_size, url, storage_provider,
COALESCE(storage_channel_id::text, ''), COALESCE(storage_channel_key, ''),
COALESCE(object_key, ''), COALESCE(access_scope, 'private'),
COALESCE(local_path, ''), expires_at, expired_at, ref_count, created_at, updated_at`,
input.SHA256,
input.ContentType,
input.ByteSize,
input.URL,
input.StorageProvider,
input.StorageChannelID,
input.StorageChannelKey,
input.ObjectKey,
input.AccessScope,
input.LocalPath,
input.ExpiresAt,
))
@@ -110,6 +132,10 @@ func scanRequestAsset(scanner interface{ Scan(dest ...any) error }) (RequestAsse
&asset.ByteSize,
&asset.URL,
&asset.StorageProvider,
&asset.StorageChannelID,
&asset.StorageChannelKey,
&asset.ObjectKey,
&asset.AccessScope,
&localPath,
&expiresAt,
&expiredAt,
apps/api/internal/store/task_history.go
+23 -13
View File
@@ -13,17 +13,20 @@ const (
)
type TaskCallbackDelivery struct {
ID string
TaskID string
EventID string
Seq int64
CallbackURL string
Status string
Attempts int
LockToken string
EventType string
TaskStatus string
CreatedAt time.Time
ID string
TaskID string
EventID string
Seq int64
CallbackURL string
Status string
Attempts int
LockToken string
EventType string
TaskStatus string
TaskRequestID string
TaskErrorCode string
TaskErrorMessage string
CreatedAt time.Time
}
type TaskHistoryCleanupResult struct {
@@ -67,9 +70,13 @@ func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit i
}
rows, err := s.pool.Query(ctx, `
WITH picked AS (
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at,
COALESCE(task.request_id, '') AS task_request_id,
COALESCE(task.error_code, '') AS task_error_code,
COALESCE(task.error_message, task.error, '') AS task_error_message
FROM gateway_task_callback_outbox outbox
JOIN gateway_task_events event ON event.id = outbox.event_id
JOIN gateway_tasks task ON task.id = outbox.task_id
WHERE outbox.created_at >= COALESCE((
SELECT applied_at
FROM schema_migrations
@@ -101,7 +108,7 @@ WHERE outbox.id = picked.id
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
picked.created_at`,
picked.created_at, picked.task_request_id, picked.task_error_code, picked.task_error_message`,
workerID, limit, int(staleAfter/time.Second))
if err != nil {
return nil, err
@@ -122,6 +129,9 @@ RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text,
&item.EventType,
&item.TaskStatus,
&item.CreatedAt,
&item.TaskRequestID,
&item.TaskErrorCode,
&item.TaskErrorMessage,
); err != nil {
return nil, err
}
apps/api/internal/store/tasks_runtime.go
+23 -1
View File
@@ -11,6 +11,7 @@ import (
"unicode/utf8"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/publicerror"
"github.com/jackc/pgx/v5"
)
@@ -544,12 +545,14 @@ WHERE id = $1::uuid
}
func (s *Store) FailQueuedTask(ctx context.Context, taskID string, code string, message string) (GatewayTask, error) {
publicErrorJSON := encodePublicErrorSnapshot(code, message, 0, true, "", taskID)
tag, err := s.pool.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
error = NULL,
error_code = NULLIF($2, ''),
error_message = NULLIF($3, ''),
public_error = $4::jsonb,
billing_status = CASE
WHEN run_mode NOT IN ('production', 'acceptance', 'acceptance_canary') OR gateway_user_id IS NULL THEN 'not_required'
ELSE 'released'
@@ -563,7 +566,7 @@ SET status = 'failed',
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048))
AND status = 'queued'`, taskID, strings.TrimSpace(code), truncateUTF8Bytes(message, 2048), string(publicErrorJSON))
if err != nil {
return GatewayTask{}, err
}
@@ -686,6 +689,7 @@ SET status = 'queued',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
updated_at = now()
WHERE id = $1::uuid
AND status IN ('queued', 'running')
@@ -722,6 +726,7 @@ SET status = 'queued',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
updated_at = now()
WHERE task.id = $1::uuid
AND task.status IN ('queued', 'running')
@@ -1371,6 +1376,7 @@ SELECT a.id::text, a.task_id::text, a.attempt_no,
a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb),
COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''),
COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''),
COALESCE(a.public_error, '{}'::jsonb),
COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''),
a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''),
a.started_at, COALESCE(a.finished_at::text, '')
@@ -1403,6 +1409,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
var requestBytes []byte
var responseBytes []byte
var pricingSnapshotBytes []byte
var publicErrorBytes []byte
if err := scanner.Scan(
&item.ID,
&item.TaskID,
@@ -1430,6 +1437,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
&item.ResponseDurationMS,
&item.ErrorCode,
&item.ErrorMessage,
&publicErrorBytes,
&pricingSnapshotBytes,
&item.RequestFingerprint,
&item.UpstreamSubmissionStatus,
@@ -1444,6 +1452,12 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
item.RequestSnapshot = decodeObject(requestBytes)
item.ResponseSnapshot = decodeObject(responseBytes)
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
if len(publicErrorBytes) > 0 {
var snapshot publicerror.Error
if json.Unmarshal(publicErrorBytes, &snapshot) == nil && snapshot.Code != "" {
item.PublicError = &snapshot
}
}
enrichTaskAttemptFromMetrics(&item)
return item, nil
}
@@ -1534,6 +1548,7 @@ func (s *Store) FinishTaskAttempt(ctx context.Context, input FinishTaskAttemptIn
}
usageJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptUsage(input.Usage)))
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(minimalTaskAttemptMetrics(input.Metrics)))
publicErrorJSON := encodePublicErrorSnapshot(input.ErrorCode, input.ErrorMessage, statusCode, input.Retryable, input.RequestID, "")
_, err := s.pool.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = $2::text,
@@ -1550,6 +1565,7 @@ SET status = $2::text,
response_duration_ms = $8,
error_code = NULLIF($9::text, ''),
error_message = NULLIF(left($10::text, 2048), ''),
public_error = CASE WHEN $2::text = 'failed' THEN $14::jsonb ELSE NULL END,
upstream_submission_status = COALESCE(NULLIF($13::text, ''), upstream_submission_status),
upstream_submission_updated_at = CASE
WHEN NULLIF($13::text, '') IS NULL THEN upstream_submission_updated_at
@@ -1570,6 +1586,7 @@ WHERE id = $1::uuid`,
usageJSON,
metricsJSON,
input.UpstreamSubmissionStatus,
string(publicErrorJSON),
)
return err
}
@@ -1651,6 +1668,7 @@ SET status = 'succeeded',
response_duration_ms = $5,
error_code = NULL,
error_message = NULL,
public_error = NULL,
finished_at = now()
WHERE id = $1::uuid`,
input.AttemptID, input.RequestID,
@@ -1687,6 +1705,7 @@ SET status = 'succeeded',
error = NULL,
error_code = NULL,
error_message = NULL,
public_error = NULL,
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
@@ -2049,6 +2068,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
metricsJSON, _ := json.Marshal(sanitizeJSONForStorage(emptyObjectIfNil(input.Metrics)))
resultJSON, _ := json.Marshal(minimalTaskResult(nil))
message := truncateUTF8Bytes(input.Message, 2048)
publicErrorJSON := encodePublicErrorSnapshot(input.Code, message, 0, true, input.RequestID, input.TaskID)
finalizedAdmission := false
err := s.beginTransaction(ctx, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
@@ -2057,6 +2077,7 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
error = NULL,
error_code = NULLIF($3::text, ''),
error_message = NULLIF($2::text, ''),
public_error = $11::jsonb,
request_id = NULLIF($4::text, ''),
metrics = $5::jsonb,
response_started_at = $6::timestamptz,
@@ -2090,6 +2111,7 @@ WHERE id = $1::uuid
input.ResponseDurationMS,
string(resultJSON),
input.ExecutionToken,
string(publicErrorJSON),
)
if err != nil {
return err