perf(storage): 极简化任务历史并增加保留治理

停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。

增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。

验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
2026-07-24 18:23:22 +08:00
parent 09375bfae7
commit 810dcfeee6
46 changed files with 2246 additions and 458 deletions
@@ -0,0 +1,11 @@
ALTER TABLE gateway_task_events
ADD COLUMN IF NOT EXISTS platform_id uuid;
ALTER TABLE gateway_task_attempts
ADD COLUMN IF NOT EXISTS status_code integer;
ALTER TABLE gateway_task_callback_outbox
ADD COLUMN IF NOT EXISTS locked_by text,
ADD COLUMN IF NOT EXISTS lock_token uuid,
ADD COLUMN IF NOT EXISTS locked_at timestamptz,
ADD COLUMN IF NOT EXISTS failed_at timestamptz;
@@ -0,0 +1,68 @@
-- easyai:migration:no-transaction
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_retention
ON gateway_tasks(finished_at, id)
WHERE status IN ('succeeded', 'failed', 'cancelled');
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_attempts_retention
ON gateway_task_attempts((COALESCE(finished_at, started_at)), id)
WHERE status <> 'running';
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_retention
ON gateway_task_events(created_at, id);
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_param_logs_retention
ON gateway_task_param_preprocessing_logs(created_at, id);
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_minimal_compaction
ON gateway_tasks(updated_at, id)
WHERE COALESCE(normalized_request, '{}'::jsonb) <> '{}'::jsonb
OR compatibility_public_id IS NOT NULL
OR compatibility_submit_http_status IS NOT NULL
OR COALESCE(compatibility_submit_headers, '{}'::jsonb) <> '{}'::jsonb
OR COALESCE(compatibility_submit_body, '{}'::jsonb) <> '{}'::jsonb
OR result ?| ARRAY[
'raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse',
'submit', 'file_retrieve', 'fileRetrieve', 'upstream_task_id', 'remote_task_id'
]
OR error IS NOT NULL
OR octet_length(COALESCE(error_message, '')) > 2048;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_checkpoint_compaction
ON gateway_tasks(updated_at, id)
WHERE COALESCE(remote_task_payload, '{}'::jsonb) <> '{}'::jsonb;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_attempts_minimal_compaction
ON gateway_task_attempts(started_at, id)
WHERE request_snapshot <> '{}'::jsonb
OR COALESCE(response_snapshot, '{}'::jsonb) <> '{}'::jsonb
OR COALESCE(usage, '{}'::jsonb) <> '{}'::jsonb
OR COALESCE(metrics, '{}'::jsonb) <> '{}'::jsonb
OR COALESCE(pricing_snapshot, '{}'::jsonb) <> '{}'::jsonb
OR request_fingerprint IS NOT NULL;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_minimal_compaction
ON gateway_task_events(created_at, id)
WHERE payload <> '{}'::jsonb
OR phase IS NOT NULL
OR COALESCE(progress, 0) <> 0
OR message IS NOT NULL;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_param_logs_minimal_compaction
ON gateway_task_param_preprocessing_logs(created_at, id)
WHERE actual_input <> '{}'::jsonb
OR converted_output <> '{}'::jsonb
OR model_snapshot <> '{}'::jsonb
OR octet_length(changes::text) > 1024;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_cloned_voices_minimal_compaction
ON gateway_cloned_voices(updated_at, id)
WHERE metadata ?| ARRAY['raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse', 'request'];
@@ -0,0 +1,4 @@
-- easyai:migration:no-transaction
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_platform_created
ON gateway_task_events(platform_id, created_at DESC)
WHERE platform_id IS NOT NULL;
@@ -0,0 +1,24 @@
-- easyai:migration:no-transaction
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_stale
ON gateway_task_callback_outbox(status, locked_at)
WHERE status = 'processing';
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_delivered_retention
ON gateway_task_callback_outbox(delivered_at, id)
WHERE status = 'delivered';
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_failed_retention
ON gateway_task_callback_outbox(failed_at, id)
WHERE status = 'failed';
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_payload_compaction
ON gateway_task_callback_outbox(created_at, id)
WHERE payload <> '{}'::jsonb;
-- easyai:migration:statement
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_legacy
ON gateway_task_callback_outbox(created_at, id)
WHERE status IN ('pending', 'processing');