wangbo
810dcfeee6
停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON，并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。 增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭，待兼容协议和异步恢复在线验证后单独启用。 验证：Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
497 lines
15 KiB
Go
497 lines
15 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
)
|
|
|
|
const (
|
|
TaskCallbackBatchSize = 100
|
|
TaskCallbackLockTTL = 60 * time.Second
|
|
)
|
|
|
|
type TaskCallbackDelivery struct {
|
|
ID string
|
|
TaskID string
|
|
EventID string
|
|
Seq int64
|
|
CallbackURL string
|
|
Status string
|
|
Attempts int
|
|
LockToken string
|
|
EventType string
|
|
TaskStatus string
|
|
CreatedAt time.Time
|
|
}
|
|
|
|
type TaskHistoryCleanupResult struct {
|
|
CompactedTasks int64
|
|
CompactedCheckpoints int64
|
|
CompactedAttempts int64
|
|
CompactedEvents int64
|
|
CompactedCallbacks int64
|
|
RetiredCallbacks int64
|
|
CompactedParamLogs int64
|
|
CompactedClonedVoices int64
|
|
DeletedCallbacks int64
|
|
DeletedParamLogs int64
|
|
DeletedEvents int64
|
|
DeletedAttempts int64
|
|
DeletedTasks int64
|
|
}
|
|
|
|
func (result TaskHistoryCleanupResult) Total() int64 {
|
|
return result.CompactedTasks +
|
|
result.CompactedCheckpoints +
|
|
result.CompactedAttempts +
|
|
result.CompactedEvents +
|
|
result.CompactedCallbacks +
|
|
result.RetiredCallbacks +
|
|
result.CompactedParamLogs +
|
|
result.CompactedClonedVoices +
|
|
result.DeletedCallbacks +
|
|
result.DeletedParamLogs +
|
|
result.DeletedEvents +
|
|
result.DeletedAttempts +
|
|
result.DeletedTasks
|
|
}
|
|
|
|
func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit int, staleAfter time.Duration) ([]TaskCallbackDelivery, error) {
|
|
if limit <= 0 || limit > TaskCallbackBatchSize {
|
|
limit = TaskCallbackBatchSize
|
|
}
|
|
if staleAfter <= 0 {
|
|
staleAfter = TaskCallbackLockTTL
|
|
}
|
|
rows, err := s.pool.Query(ctx, `
|
|
WITH picked AS (
|
|
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at
|
|
FROM gateway_task_callback_outbox outbox
|
|
JOIN gateway_task_events event ON event.id = outbox.event_id
|
|
WHERE outbox.created_at >= COALESCE((
|
|
SELECT applied_at
|
|
FROM schema_migrations
|
|
WHERE version = '0083_task_history_minimal_storage'
|
|
), now())
|
|
AND (
|
|
(
|
|
outbox.status = 'pending'
|
|
AND outbox.next_attempt_at <= now()
|
|
)
|
|
OR (
|
|
outbox.status = 'processing'
|
|
AND outbox.locked_at < now() - ($3::int * interval '1 second')
|
|
)
|
|
)
|
|
ORDER BY outbox.next_attempt_at, outbox.created_at
|
|
LIMIT $2
|
|
FOR UPDATE OF outbox SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_callback_outbox outbox
|
|
SET status = 'processing',
|
|
attempts = outbox.attempts + 1,
|
|
locked_by = $1,
|
|
lock_token = gen_random_uuid(),
|
|
locked_at = now(),
|
|
updated_at = now()
|
|
FROM picked
|
|
WHERE outbox.id = picked.id
|
|
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
|
|
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
|
|
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
|
|
picked.created_at`,
|
|
workerID, limit, int(staleAfter/time.Second))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
items := make([]TaskCallbackDelivery, 0)
|
|
for rows.Next() {
|
|
var item TaskCallbackDelivery
|
|
if err := rows.Scan(
|
|
&item.ID,
|
|
&item.TaskID,
|
|
&item.EventID,
|
|
&item.Seq,
|
|
&item.CallbackURL,
|
|
&item.Status,
|
|
&item.Attempts,
|
|
&item.LockToken,
|
|
&item.EventType,
|
|
&item.TaskStatus,
|
|
&item.CreatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
items = append(items, item)
|
|
}
|
|
return items, rows.Err()
|
|
}
|
|
|
|
func (s *Store) MarkTaskCallbackDelivered(ctx context.Context, item TaskCallbackDelivery) error {
|
|
_, err := s.pool.Exec(ctx, `
|
|
UPDATE gateway_task_callback_outbox
|
|
SET status = 'delivered',
|
|
delivered_at = now(),
|
|
failed_at = NULL,
|
|
last_error = NULL,
|
|
locked_by = NULL,
|
|
lock_token = NULL,
|
|
locked_at = NULL,
|
|
updated_at = now()
|
|
WHERE id = $1::uuid
|
|
AND status = 'processing'
|
|
AND lock_token = $2::uuid`, item.ID, item.LockToken)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) MarkTaskCallbackFailed(ctx context.Context, item TaskCallbackDelivery, retry bool, nextAttemptAt time.Time, message string) error {
|
|
status := "failed"
|
|
if retry {
|
|
status = "pending"
|
|
}
|
|
_, err := s.pool.Exec(ctx, `
|
|
UPDATE gateway_task_callback_outbox
|
|
SET status = $3,
|
|
next_attempt_at = CASE WHEN $3 = 'pending' THEN $4::timestamptz ELSE next_attempt_at END,
|
|
failed_at = CASE WHEN $3 = 'failed' THEN now() ELSE NULL END,
|
|
last_error = NULLIF(left($5, 2048), ''),
|
|
locked_by = NULL,
|
|
lock_token = NULL,
|
|
locked_at = NULL,
|
|
updated_at = now()
|
|
WHERE id = $1::uuid
|
|
AND status = 'processing'
|
|
AND lock_token = $2::uuid`,
|
|
item.ID, item.LockToken, status, nextAttemptAt, message)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) CleanupTaskHistory(ctx context.Context, analysisCutoff time.Time, taskCutoff time.Time, batchSize int) (TaskHistoryCleanupResult, error) {
|
|
if batchSize < 1 {
|
|
batchSize = 1000
|
|
}
|
|
var result TaskHistoryCleanupResult
|
|
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
|
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
|
return err
|
|
}
|
|
steps := []struct {
|
|
target *int64
|
|
sql string
|
|
args []any
|
|
}{
|
|
{&result.CompactedTasks, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_tasks
|
|
WHERE COALESCE(normalized_request, '{}'::jsonb) <> '{}'::jsonb
|
|
OR compatibility_public_id IS NOT NULL
|
|
OR compatibility_submit_http_status IS NOT NULL
|
|
OR COALESCE(compatibility_submit_headers, '{}'::jsonb) <> '{}'::jsonb
|
|
OR COALESCE(compatibility_submit_body, '{}'::jsonb) <> '{}'::jsonb
|
|
OR result ?| ARRAY[
|
|
'raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse',
|
|
'submit', 'file_retrieve', 'fileRetrieve', 'upstream_task_id', 'remote_task_id'
|
|
]
|
|
OR error IS NOT NULL
|
|
OR octet_length(COALESCE(error_message, '')) > 2048
|
|
ORDER BY updated_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_tasks task
|
|
SET normalized_request = '{}'::jsonb,
|
|
compatibility_public_id = NULL,
|
|
compatibility_submit_http_status = NULL,
|
|
compatibility_submit_headers = '{}'::jsonb,
|
|
compatibility_submit_body = '{}'::jsonb,
|
|
result = COALESCE(task.result, '{}'::jsonb)
|
|
- 'raw' - 'raw_data' - 'rawData'
|
|
- 'provider_response' - 'providerResponse'
|
|
- 'submit' - 'file_retrieve' - 'fileRetrieve'
|
|
- 'upstream_task_id' - 'remote_task_id',
|
|
error = NULL,
|
|
error_message = CASE
|
|
WHEN octet_length(COALESCE(task.error_message, '')) > 2048 THEN left(task.error_message, 512)
|
|
ELSE task.error_message
|
|
END,
|
|
remote_task_payload = CASE
|
|
WHEN task.status IN ('succeeded', 'failed', 'cancelled') THEN '{}'::jsonb
|
|
ELSE (
|
|
SELECT COALESCE(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb)
|
|
FROM jsonb_each(COALESCE(task.remote_task_payload, '{}'::jsonb)) entry
|
|
WHERE entry.key IN (
|
|
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
|
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
|
)
|
|
)
|
|
END
|
|
FROM picked
|
|
WHERE task.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedCheckpoints, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_tasks
|
|
WHERE COALESCE(remote_task_payload, '{}'::jsonb) <> '{}'::jsonb
|
|
AND (
|
|
status IN ('succeeded', 'failed', 'cancelled')
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM jsonb_object_keys(COALESCE(remote_task_payload, '{}'::jsonb)) AS key
|
|
WHERE key NOT IN (
|
|
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
|
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
|
)
|
|
)
|
|
)
|
|
ORDER BY updated_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_tasks task
|
|
SET remote_task_payload = CASE
|
|
WHEN task.status IN ('succeeded', 'failed', 'cancelled') THEN '{}'::jsonb
|
|
ELSE (
|
|
SELECT COALESCE(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb)
|
|
FROM jsonb_each(COALESCE(task.remote_task_payload, '{}'::jsonb)) entry
|
|
WHERE entry.key IN (
|
|
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
|
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
|
)
|
|
)
|
|
END
|
|
FROM picked
|
|
WHERE task.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedAttempts, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_attempts
|
|
WHERE request_snapshot <> '{}'::jsonb
|
|
OR COALESCE(response_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
|
OR COALESCE(usage, '{}'::jsonb) <> '{}'::jsonb
|
|
OR COALESCE(metrics, '{}'::jsonb) <> '{}'::jsonb
|
|
OR COALESCE(pricing_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
|
OR request_fingerprint IS NOT NULL
|
|
ORDER BY started_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_attempts attempt
|
|
SET request_snapshot = '{}'::jsonb,
|
|
response_snapshot = '{}'::jsonb,
|
|
usage = '{}'::jsonb,
|
|
metrics = '{}'::jsonb,
|
|
pricing_snapshot = '{}'::jsonb,
|
|
request_fingerprint = NULL,
|
|
error_message = NULLIF(left(error_message, 2048), '')
|
|
FROM picked
|
|
WHERE attempt.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedEvents, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_events
|
|
WHERE payload <> '{}'::jsonb
|
|
OR phase IS NOT NULL
|
|
OR COALESCE(progress, 0) <> 0
|
|
OR message IS NOT NULL
|
|
ORDER BY created_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_events event
|
|
SET platform_id = CASE
|
|
WHEN event.platform_id IS NULL
|
|
AND event.payload->>'platformId' ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
|
|
THEN (event.payload->>'platformId')::uuid
|
|
ELSE event.platform_id
|
|
END,
|
|
phase = NULL,
|
|
progress = 0,
|
|
message = NULL,
|
|
payload = '{}'::jsonb
|
|
FROM picked
|
|
WHERE event.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedCallbacks, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_callback_outbox
|
|
WHERE payload <> '{}'::jsonb
|
|
ORDER BY created_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_callback_outbox outbox
|
|
SET payload = '{}'::jsonb
|
|
FROM picked
|
|
WHERE outbox.id = picked.id`, []any{batchSize}},
|
|
{&result.RetiredCallbacks, `
|
|
WITH migration_boundary AS (
|
|
SELECT applied_at
|
|
FROM schema_migrations
|
|
WHERE version = '0083_task_history_minimal_storage'
|
|
), picked AS (
|
|
SELECT outbox.id
|
|
FROM gateway_task_callback_outbox outbox
|
|
CROSS JOIN migration_boundary boundary
|
|
WHERE outbox.created_at < boundary.applied_at
|
|
AND outbox.status IN ('pending', 'processing')
|
|
ORDER BY outbox.created_at
|
|
LIMIT $1
|
|
FOR UPDATE OF outbox SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_callback_outbox outbox
|
|
SET status = 'failed',
|
|
failed_at = outbox.created_at,
|
|
last_error = 'legacy_callback_not_replayed',
|
|
locked_by = NULL,
|
|
lock_token = NULL,
|
|
locked_at = NULL,
|
|
updated_at = now()
|
|
FROM picked
|
|
WHERE outbox.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedParamLogs, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_param_preprocessing_logs
|
|
WHERE actual_input <> '{}'::jsonb
|
|
OR converted_output <> '{}'::jsonb
|
|
OR model_snapshot <> '{}'::jsonb
|
|
OR octet_length(changes::text) > 1024
|
|
ORDER BY created_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_task_param_preprocessing_logs log
|
|
SET actual_input = '{}'::jsonb,
|
|
converted_output = '{}'::jsonb,
|
|
model_snapshot = '{}'::jsonb,
|
|
changes = CASE WHEN octet_length(log.changes::text) > 1024 THEN '[]'::jsonb ELSE log.changes END
|
|
FROM picked
|
|
WHERE log.id = picked.id`, []any{batchSize}},
|
|
{&result.CompactedClonedVoices, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_cloned_voices
|
|
WHERE metadata ?| ARRAY['raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse', 'request']
|
|
ORDER BY updated_at
|
|
LIMIT $1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE gateway_cloned_voices voice
|
|
SET metadata = COALESCE(voice.metadata, '{}'::jsonb)
|
|
- 'raw' - 'raw_data' - 'rawData'
|
|
- 'provider_response' - 'providerResponse' - 'request',
|
|
updated_at = now()
|
|
FROM picked
|
|
WHERE voice.id = picked.id`, []any{batchSize}},
|
|
{&result.DeletedCallbacks, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_callback_outbox
|
|
WHERE (status = 'delivered' AND delivered_at < now() - interval '24 hours')
|
|
OR (status = 'failed' AND COALESCE(failed_at, updated_at) < $1::timestamptz)
|
|
ORDER BY updated_at
|
|
LIMIT $2
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
DELETE FROM gateway_task_callback_outbox outbox
|
|
USING picked
|
|
WHERE outbox.id = picked.id`, []any{analysisCutoff, batchSize}},
|
|
{&result.DeletedParamLogs, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_param_preprocessing_logs
|
|
WHERE created_at < $1::timestamptz
|
|
ORDER BY created_at
|
|
LIMIT $2
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
DELETE FROM gateway_task_param_preprocessing_logs log
|
|
USING picked
|
|
WHERE log.id = picked.id`, []any{analysisCutoff, batchSize}},
|
|
{&result.DeletedEvents, `
|
|
WITH picked AS (
|
|
SELECT event.id
|
|
FROM gateway_task_events event
|
|
WHERE event.created_at < $1::timestamptz
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM gateway_task_callback_outbox outbox
|
|
WHERE outbox.event_id = event.id
|
|
AND outbox.status IN ('pending', 'processing')
|
|
)
|
|
ORDER BY event.created_at
|
|
LIMIT $2
|
|
FOR UPDATE OF event SKIP LOCKED
|
|
)
|
|
DELETE FROM gateway_task_events event
|
|
USING picked
|
|
WHERE event.id = picked.id`, []any{analysisCutoff, batchSize}},
|
|
{&result.DeletedAttempts, `
|
|
WITH picked AS (
|
|
SELECT id
|
|
FROM gateway_task_attempts
|
|
WHERE COALESCE(finished_at, started_at) < $1::timestamptz
|
|
AND status <> 'running'
|
|
ORDER BY COALESCE(finished_at, started_at)
|
|
LIMIT $2
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
DELETE FROM gateway_task_attempts attempt
|
|
USING picked
|
|
WHERE attempt.id = picked.id`, []any{analysisCutoff, batchSize}},
|
|
{&result.DeletedTasks, `
|
|
WITH picked AS (
|
|
SELECT task.id
|
|
FROM gateway_tasks task
|
|
WHERE task.finished_at < $1::timestamptz
|
|
AND task.status IN ('succeeded', 'failed', 'cancelled')
|
|
AND task.billing_status IN ('settled', 'released', 'not_required')
|
|
AND (task.execution_lease_expires_at IS NULL OR task.execution_lease_expires_at <= now())
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM settlement_outbox settlement
|
|
WHERE settlement.task_id = task.id
|
|
AND settlement.status IN ('pending', 'processing', 'retryable_failed', 'manual_review')
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM gateway_task_callback_outbox callback
|
|
WHERE callback.task_id = task.id
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM gateway_concurrency_leases lease
|
|
WHERE lease.task_id = task.id
|
|
AND lease.released_at IS NULL
|
|
AND lease.expires_at > now()
|
|
)
|
|
AND NOT EXISTS (
|
|
SELECT 1
|
|
FROM gateway_rate_limit_reservations reservation
|
|
WHERE reservation.task_id = task.id
|
|
AND reservation.status = 'reserved'
|
|
)
|
|
ORDER BY task.finished_at
|
|
LIMIT $2
|
|
FOR UPDATE OF task SKIP LOCKED
|
|
)
|
|
DELETE FROM gateway_tasks task
|
|
USING picked
|
|
WHERE task.id = picked.id`, []any{taskCutoff, batchSize}},
|
|
}
|
|
for _, step := range steps {
|
|
tag, err := tx.Exec(ctx, step.sql, step.args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
*step.target = tag.RowsAffected()
|
|
}
|
|
return nil
|
|
})
|
|
return result, err
|
|
}
|