perf(storage): 极简化任务历史并增加保留治理
停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON，并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。 增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭，待兼容协议和异步恢复在线验证后单独启用。 验证：Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
@@ -0,0 +1,496 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const (
|
||||
TaskCallbackBatchSize = 100
|
||||
TaskCallbackLockTTL = 60 * time.Second
|
||||
)
|
||||
|
||||
type TaskCallbackDelivery struct {
|
||||
ID string
|
||||
TaskID string
|
||||
EventID string
|
||||
Seq int64
|
||||
CallbackURL string
|
||||
Status string
|
||||
Attempts int
|
||||
LockToken string
|
||||
EventType string
|
||||
TaskStatus string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type TaskHistoryCleanupResult struct {
|
||||
CompactedTasks int64
|
||||
CompactedCheckpoints int64
|
||||
CompactedAttempts int64
|
||||
CompactedEvents int64
|
||||
CompactedCallbacks int64
|
||||
RetiredCallbacks int64
|
||||
CompactedParamLogs int64
|
||||
CompactedClonedVoices int64
|
||||
DeletedCallbacks int64
|
||||
DeletedParamLogs int64
|
||||
DeletedEvents int64
|
||||
DeletedAttempts int64
|
||||
DeletedTasks int64
|
||||
}
|
||||
|
||||
func (result TaskHistoryCleanupResult) Total() int64 {
|
||||
return result.CompactedTasks +
|
||||
result.CompactedCheckpoints +
|
||||
result.CompactedAttempts +
|
||||
result.CompactedEvents +
|
||||
result.CompactedCallbacks +
|
||||
result.RetiredCallbacks +
|
||||
result.CompactedParamLogs +
|
||||
result.CompactedClonedVoices +
|
||||
result.DeletedCallbacks +
|
||||
result.DeletedParamLogs +
|
||||
result.DeletedEvents +
|
||||
result.DeletedAttempts +
|
||||
result.DeletedTasks
|
||||
}
|
||||
|
||||
func (s *Store) ClaimTaskCallbacks(ctx context.Context, workerID string, limit int, staleAfter time.Duration) ([]TaskCallbackDelivery, error) {
|
||||
if limit <= 0 || limit > TaskCallbackBatchSize {
|
||||
limit = TaskCallbackBatchSize
|
||||
}
|
||||
if staleAfter <= 0 {
|
||||
staleAfter = TaskCallbackLockTTL
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
WITH picked AS (
|
||||
SELECT outbox.id, event.event_type, COALESCE(event.status, '') AS task_status, event.created_at
|
||||
FROM gateway_task_callback_outbox outbox
|
||||
JOIN gateway_task_events event ON event.id = outbox.event_id
|
||||
WHERE outbox.created_at >= COALESCE((
|
||||
SELECT applied_at
|
||||
FROM schema_migrations
|
||||
WHERE version = '0083_task_history_minimal_storage'
|
||||
), now())
|
||||
AND (
|
||||
(
|
||||
outbox.status = 'pending'
|
||||
AND outbox.next_attempt_at <= now()
|
||||
)
|
||||
OR (
|
||||
outbox.status = 'processing'
|
||||
AND outbox.locked_at < now() - ($3::int * interval '1 second')
|
||||
)
|
||||
)
|
||||
ORDER BY outbox.next_attempt_at, outbox.created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE OF outbox SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_callback_outbox outbox
|
||||
SET status = 'processing',
|
||||
attempts = outbox.attempts + 1,
|
||||
locked_by = $1,
|
||||
lock_token = gen_random_uuid(),
|
||||
locked_at = now(),
|
||||
updated_at = now()
|
||||
FROM picked
|
||||
WHERE outbox.id = picked.id
|
||||
RETURNING outbox.id::text, outbox.task_id::text, COALESCE(outbox.event_id::text, ''),
|
||||
outbox.seq, outbox.callback_url, outbox.status, outbox.attempts,
|
||||
COALESCE(outbox.lock_token::text, ''), picked.event_type, picked.task_status,
|
||||
picked.created_at`,
|
||||
workerID, limit, int(staleAfter/time.Second))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]TaskCallbackDelivery, 0)
|
||||
for rows.Next() {
|
||||
var item TaskCallbackDelivery
|
||||
if err := rows.Scan(
|
||||
&item.ID,
|
||||
&item.TaskID,
|
||||
&item.EventID,
|
||||
&item.Seq,
|
||||
&item.CallbackURL,
|
||||
&item.Status,
|
||||
&item.Attempts,
|
||||
&item.LockToken,
|
||||
&item.EventType,
|
||||
&item.TaskStatus,
|
||||
&item.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) MarkTaskCallbackDelivered(ctx context.Context, item TaskCallbackDelivery) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_callback_outbox
|
||||
SET status = 'delivered',
|
||||
delivered_at = now(),
|
||||
failed_at = NULL,
|
||||
last_error = NULL,
|
||||
locked_by = NULL,
|
||||
lock_token = NULL,
|
||||
locked_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND lock_token = $2::uuid`, item.ID, item.LockToken)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) MarkTaskCallbackFailed(ctx context.Context, item TaskCallbackDelivery, retry bool, nextAttemptAt time.Time, message string) error {
|
||||
status := "failed"
|
||||
if retry {
|
||||
status = "pending"
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_task_callback_outbox
|
||||
SET status = $3,
|
||||
next_attempt_at = CASE WHEN $3 = 'pending' THEN $4::timestamptz ELSE next_attempt_at END,
|
||||
failed_at = CASE WHEN $3 = 'failed' THEN now() ELSE NULL END,
|
||||
last_error = NULLIF(left($5, 2048), ''),
|
||||
locked_by = NULL,
|
||||
lock_token = NULL,
|
||||
locked_at = NULL,
|
||||
updated_at = now()
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND lock_token = $2::uuid`,
|
||||
item.ID, item.LockToken, status, nextAttemptAt, message)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) CleanupTaskHistory(ctx context.Context, analysisCutoff time.Time, taskCutoff time.Time, batchSize int) (TaskHistoryCleanupResult, error) {
|
||||
if batchSize < 1 {
|
||||
batchSize = 1000
|
||||
}
|
||||
var result TaskHistoryCleanupResult
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SET LOCAL lock_timeout = '5s'; SET LOCAL statement_timeout = '5s'`); err != nil {
|
||||
return err
|
||||
}
|
||||
steps := []struct {
|
||||
target *int64
|
||||
sql string
|
||||
args []any
|
||||
}{
|
||||
{&result.CompactedTasks, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_tasks
|
||||
WHERE COALESCE(normalized_request, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR compatibility_public_id IS NOT NULL
|
||||
OR compatibility_submit_http_status IS NOT NULL
|
||||
OR COALESCE(compatibility_submit_headers, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(compatibility_submit_body, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR result ?| ARRAY[
|
||||
'raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse',
|
||||
'submit', 'file_retrieve', 'fileRetrieve', 'upstream_task_id', 'remote_task_id'
|
||||
]
|
||||
OR error IS NOT NULL
|
||||
OR octet_length(COALESCE(error_message, '')) > 2048
|
||||
ORDER BY updated_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_tasks task
|
||||
SET normalized_request = '{}'::jsonb,
|
||||
compatibility_public_id = NULL,
|
||||
compatibility_submit_http_status = NULL,
|
||||
compatibility_submit_headers = '{}'::jsonb,
|
||||
compatibility_submit_body = '{}'::jsonb,
|
||||
result = COALESCE(task.result, '{}'::jsonb)
|
||||
- 'raw' - 'raw_data' - 'rawData'
|
||||
- 'provider_response' - 'providerResponse'
|
||||
- 'submit' - 'file_retrieve' - 'fileRetrieve'
|
||||
- 'upstream_task_id' - 'remote_task_id',
|
||||
error = NULL,
|
||||
error_message = CASE
|
||||
WHEN octet_length(COALESCE(task.error_message, '')) > 2048 THEN left(task.error_message, 512)
|
||||
ELSE task.error_message
|
||||
END,
|
||||
remote_task_payload = CASE
|
||||
WHEN task.status IN ('succeeded', 'failed', 'cancelled') THEN '{}'::jsonb
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb)
|
||||
FROM jsonb_each(COALESCE(task.remote_task_payload, '{}'::jsonb)) entry
|
||||
WHERE entry.key IN (
|
||||
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
||||
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
||||
)
|
||||
)
|
||||
END
|
||||
FROM picked
|
||||
WHERE task.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedCheckpoints, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_tasks
|
||||
WHERE COALESCE(remote_task_payload, '{}'::jsonb) <> '{}'::jsonb
|
||||
AND (
|
||||
status IN ('succeeded', 'failed', 'cancelled')
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM jsonb_object_keys(COALESCE(remote_task_payload, '{}'::jsonb)) AS key
|
||||
WHERE key NOT IN (
|
||||
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
||||
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
||||
)
|
||||
)
|
||||
)
|
||||
ORDER BY updated_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_tasks task
|
||||
SET remote_task_payload = CASE
|
||||
WHEN task.status IN ('succeeded', 'failed', 'cancelled') THEN '{}'::jsonb
|
||||
ELSE (
|
||||
SELECT COALESCE(jsonb_object_agg(entry.key, entry.value), '{}'::jsonb)
|
||||
FROM jsonb_each(COALESCE(task.remote_task_payload, '{}'::jsonb)) entry
|
||||
WHERE entry.key IN (
|
||||
'endpoint', 'pollEndpoint', 'phase', 'mode', 'taskType',
|
||||
'targetResolution', 'imageToken', 'receipt', 'cleanupElementIds'
|
||||
)
|
||||
)
|
||||
END
|
||||
FROM picked
|
||||
WHERE task.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedAttempts, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_attempts
|
||||
WHERE request_snapshot <> '{}'::jsonb
|
||||
OR COALESCE(response_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(usage, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(metrics, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(pricing_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR request_fingerprint IS NOT NULL
|
||||
ORDER BY started_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_attempts attempt
|
||||
SET request_snapshot = '{}'::jsonb,
|
||||
response_snapshot = '{}'::jsonb,
|
||||
usage = '{}'::jsonb,
|
||||
metrics = '{}'::jsonb,
|
||||
pricing_snapshot = '{}'::jsonb,
|
||||
request_fingerprint = NULL,
|
||||
error_message = NULLIF(left(error_message, 2048), '')
|
||||
FROM picked
|
||||
WHERE attempt.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedEvents, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_events
|
||||
WHERE payload <> '{}'::jsonb
|
||||
OR phase IS NOT NULL
|
||||
OR COALESCE(progress, 0) <> 0
|
||||
OR message IS NOT NULL
|
||||
ORDER BY created_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_events event
|
||||
SET platform_id = CASE
|
||||
WHEN event.platform_id IS NULL
|
||||
AND event.payload->>'platformId' ~* '^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'
|
||||
THEN (event.payload->>'platformId')::uuid
|
||||
ELSE event.platform_id
|
||||
END,
|
||||
phase = NULL,
|
||||
progress = 0,
|
||||
message = NULL,
|
||||
payload = '{}'::jsonb
|
||||
FROM picked
|
||||
WHERE event.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedCallbacks, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_callback_outbox
|
||||
WHERE payload <> '{}'::jsonb
|
||||
ORDER BY created_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_callback_outbox outbox
|
||||
SET payload = '{}'::jsonb
|
||||
FROM picked
|
||||
WHERE outbox.id = picked.id`, []any{batchSize}},
|
||||
{&result.RetiredCallbacks, `
|
||||
WITH migration_boundary AS (
|
||||
SELECT applied_at
|
||||
FROM schema_migrations
|
||||
WHERE version = '0083_task_history_minimal_storage'
|
||||
), picked AS (
|
||||
SELECT outbox.id
|
||||
FROM gateway_task_callback_outbox outbox
|
||||
CROSS JOIN migration_boundary boundary
|
||||
WHERE outbox.created_at < boundary.applied_at
|
||||
AND outbox.status IN ('pending', 'processing')
|
||||
ORDER BY outbox.created_at
|
||||
LIMIT $1
|
||||
FOR UPDATE OF outbox SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_callback_outbox outbox
|
||||
SET status = 'failed',
|
||||
failed_at = outbox.created_at,
|
||||
last_error = 'legacy_callback_not_replayed',
|
||||
locked_by = NULL,
|
||||
lock_token = NULL,
|
||||
locked_at = NULL,
|
||||
updated_at = now()
|
||||
FROM picked
|
||||
WHERE outbox.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedParamLogs, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_param_preprocessing_logs
|
||||
WHERE actual_input <> '{}'::jsonb
|
||||
OR converted_output <> '{}'::jsonb
|
||||
OR model_snapshot <> '{}'::jsonb
|
||||
OR octet_length(changes::text) > 1024
|
||||
ORDER BY created_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_task_param_preprocessing_logs log
|
||||
SET actual_input = '{}'::jsonb,
|
||||
converted_output = '{}'::jsonb,
|
||||
model_snapshot = '{}'::jsonb,
|
||||
changes = CASE WHEN octet_length(log.changes::text) > 1024 THEN '[]'::jsonb ELSE log.changes END
|
||||
FROM picked
|
||||
WHERE log.id = picked.id`, []any{batchSize}},
|
||||
{&result.CompactedClonedVoices, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_cloned_voices
|
||||
WHERE metadata ?| ARRAY['raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse', 'request']
|
||||
ORDER BY updated_at
|
||||
LIMIT $1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE gateway_cloned_voices voice
|
||||
SET metadata = COALESCE(voice.metadata, '{}'::jsonb)
|
||||
- 'raw' - 'raw_data' - 'rawData'
|
||||
- 'provider_response' - 'providerResponse' - 'request',
|
||||
updated_at = now()
|
||||
FROM picked
|
||||
WHERE voice.id = picked.id`, []any{batchSize}},
|
||||
{&result.DeletedCallbacks, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_callback_outbox
|
||||
WHERE (status = 'delivered' AND delivered_at < now() - interval '24 hours')
|
||||
OR (status = 'failed' AND COALESCE(failed_at, updated_at) < $1::timestamptz)
|
||||
ORDER BY updated_at
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM gateway_task_callback_outbox outbox
|
||||
USING picked
|
||||
WHERE outbox.id = picked.id`, []any{analysisCutoff, batchSize}},
|
||||
{&result.DeletedParamLogs, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_param_preprocessing_logs
|
||||
WHERE created_at < $1::timestamptz
|
||||
ORDER BY created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM gateway_task_param_preprocessing_logs log
|
||||
USING picked
|
||||
WHERE log.id = picked.id`, []any{analysisCutoff, batchSize}},
|
||||
{&result.DeletedEvents, `
|
||||
WITH picked AS (
|
||||
SELECT event.id
|
||||
FROM gateway_task_events event
|
||||
WHERE event.created_at < $1::timestamptz
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_task_callback_outbox outbox
|
||||
WHERE outbox.event_id = event.id
|
||||
AND outbox.status IN ('pending', 'processing')
|
||||
)
|
||||
ORDER BY event.created_at
|
||||
LIMIT $2
|
||||
FOR UPDATE OF event SKIP LOCKED
|
||||
)
|
||||
DELETE FROM gateway_task_events event
|
||||
USING picked
|
||||
WHERE event.id = picked.id`, []any{analysisCutoff, batchSize}},
|
||||
{&result.DeletedAttempts, `
|
||||
WITH picked AS (
|
||||
SELECT id
|
||||
FROM gateway_task_attempts
|
||||
WHERE COALESCE(finished_at, started_at) < $1::timestamptz
|
||||
AND status <> 'running'
|
||||
ORDER BY COALESCE(finished_at, started_at)
|
||||
LIMIT $2
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
DELETE FROM gateway_task_attempts attempt
|
||||
USING picked
|
||||
WHERE attempt.id = picked.id`, []any{analysisCutoff, batchSize}},
|
||||
{&result.DeletedTasks, `
|
||||
WITH picked AS (
|
||||
SELECT task.id
|
||||
FROM gateway_tasks task
|
||||
WHERE task.finished_at < $1::timestamptz
|
||||
AND task.status IN ('succeeded', 'failed', 'cancelled')
|
||||
AND task.billing_status IN ('settled', 'released', 'not_required')
|
||||
AND (task.execution_lease_expires_at IS NULL OR task.execution_lease_expires_at <= now())
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM settlement_outbox settlement
|
||||
WHERE settlement.task_id = task.id
|
||||
AND settlement.status IN ('pending', 'processing', 'retryable_failed', 'manual_review')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_task_callback_outbox callback
|
||||
WHERE callback.task_id = task.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_concurrency_leases lease
|
||||
WHERE lease.task_id = task.id
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > now()
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM gateway_rate_limit_reservations reservation
|
||||
WHERE reservation.task_id = task.id
|
||||
AND reservation.status = 'reserved'
|
||||
)
|
||||
ORDER BY task.finished_at
|
||||
LIMIT $2
|
||||
FOR UPDATE OF task SKIP LOCKED
|
||||
)
|
||||
DELETE FROM gateway_tasks task
|
||||
USING picked
|
||||
WHERE task.id = picked.id`, []any{taskCutoff, batchSize}},
|
||||
}
|
||||
for _, step := range steps {
|
||||
tag, err := tx.Exec(ctx, step.sql, step.args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*step.target = tag.RowsAffected()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return result, err
|
||||
}
|
||||
Reference in New Issue
Block a user