perf(storage): 极简化任务历史并增加保留治理
停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。 增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。 验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE gateway_task_events
|
||||
ADD COLUMN IF NOT EXISTS platform_id uuid;
|
||||
|
||||
ALTER TABLE gateway_task_attempts
|
||||
ADD COLUMN IF NOT EXISTS status_code integer;
|
||||
|
||||
ALTER TABLE gateway_task_callback_outbox
|
||||
ADD COLUMN IF NOT EXISTS locked_by text,
|
||||
ADD COLUMN IF NOT EXISTS lock_token uuid,
|
||||
ADD COLUMN IF NOT EXISTS locked_at timestamptz,
|
||||
ADD COLUMN IF NOT EXISTS failed_at timestamptz;
|
||||
@@ -0,0 +1,68 @@
|
||||
-- easyai:migration:no-transaction
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_retention
|
||||
ON gateway_tasks(finished_at, id)
|
||||
WHERE status IN ('succeeded', 'failed', 'cancelled');
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_attempts_retention
|
||||
ON gateway_task_attempts((COALESCE(finished_at, started_at)), id)
|
||||
WHERE status <> 'running';
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_retention
|
||||
ON gateway_task_events(created_at, id);
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_param_logs_retention
|
||||
ON gateway_task_param_preprocessing_logs(created_at, id);
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_minimal_compaction
|
||||
ON gateway_tasks(updated_at, id)
|
||||
WHERE COALESCE(normalized_request, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR compatibility_public_id IS NOT NULL
|
||||
OR compatibility_submit_http_status IS NOT NULL
|
||||
OR COALESCE(compatibility_submit_headers, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(compatibility_submit_body, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR result ?| ARRAY[
|
||||
'raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse',
|
||||
'submit', 'file_retrieve', 'fileRetrieve', 'upstream_task_id', 'remote_task_id'
|
||||
]
|
||||
OR error IS NOT NULL
|
||||
OR octet_length(COALESCE(error_message, '')) > 2048;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_tasks_checkpoint_compaction
|
||||
ON gateway_tasks(updated_at, id)
|
||||
WHERE COALESCE(remote_task_payload, '{}'::jsonb) <> '{}'::jsonb;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_attempts_minimal_compaction
|
||||
ON gateway_task_attempts(started_at, id)
|
||||
WHERE request_snapshot <> '{}'::jsonb
|
||||
OR COALESCE(response_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(usage, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(metrics, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR COALESCE(pricing_snapshot, '{}'::jsonb) <> '{}'::jsonb
|
||||
OR request_fingerprint IS NOT NULL;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_minimal_compaction
|
||||
ON gateway_task_events(created_at, id)
|
||||
WHERE payload <> '{}'::jsonb
|
||||
OR phase IS NOT NULL
|
||||
OR COALESCE(progress, 0) <> 0
|
||||
OR message IS NOT NULL;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_param_logs_minimal_compaction
|
||||
ON gateway_task_param_preprocessing_logs(created_at, id)
|
||||
WHERE actual_input <> '{}'::jsonb
|
||||
OR converted_output <> '{}'::jsonb
|
||||
OR model_snapshot <> '{}'::jsonb
|
||||
OR octet_length(changes::text) > 1024;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_cloned_voices_minimal_compaction
|
||||
ON gateway_cloned_voices(updated_at, id)
|
||||
WHERE metadata ?| ARRAY['raw', 'raw_data', 'rawData', 'provider_response', 'providerResponse', 'request'];
|
||||
@@ -0,0 +1,4 @@
|
||||
-- easyai:migration:no-transaction
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_events_platform_created
|
||||
ON gateway_task_events(platform_id, created_at DESC)
|
||||
WHERE platform_id IS NOT NULL;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- easyai:migration:no-transaction
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_stale
|
||||
ON gateway_task_callback_outbox(status, locked_at)
|
||||
WHERE status = 'processing';
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_delivered_retention
|
||||
ON gateway_task_callback_outbox(delivered_at, id)
|
||||
WHERE status = 'delivered';
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_failed_retention
|
||||
ON gateway_task_callback_outbox(failed_at, id)
|
||||
WHERE status = 'failed';
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_payload_compaction
|
||||
ON gateway_task_callback_outbox(created_at, id)
|
||||
WHERE payload <> '{}'::jsonb;
|
||||
|
||||
-- easyai:migration:statement
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_gateway_task_callback_outbox_legacy
|
||||
ON gateway_task_callback_outbox(created_at, id)
|
||||
WHERE status IN ('pending', 'processing');
|
||||
Reference in New Issue
Block a user