perf(storage): 极简化任务历史并增加保留治理

停止持久化 provider 原始响应、兼容响应快照、attempt/event/outbox 重复 JSON,并由标准任务结果动态生成 Kling/Keling/Volces 兼容响应。

增加事件去重与预算、极简 callback 投递、7/30 天分批清理、安全删除条件、并发迁移索引及可实际恢复的任务域排除备份。历史清理默认关闭,待兼容协议和异步恢复在线验证后单独启用。

验证:Go 全量测试与 go vet、PostgreSQL 18 集成与实际备份恢复、迁移安全测试、bash -n、ShellCheck、Compose 配置和人工发布脚本测试均通过。
This commit is contained in:
2026-07-24 18:23:22 +08:00
parent 09375bfae7
commit 810dcfeee6
46 changed files with 2246 additions and 458 deletions
+100 -4
View File
@@ -30,7 +30,7 @@ source "$config_file"
[[ $PUBLIC_BASE_URL =~ ^https://[A-Za-z0-9.-]+$ ]] || { echo 'invalid PUBLIC_BASE_URL' >&2; exit 1; }
[[ -d $DEPLOY_DIR && -f $COMPOSE_FILE ]] || { echo 'deployment directory is unavailable' >&2; exit 1; }
for command in docker curl node flock install; do
for command in docker curl node flock install sha256sum date wc; do
command -v "$command" >/dev/null 2>&1 || { echo "$command is required" >&2; exit 1; }
done
docker info >/dev/null 2>&1 || { echo 'Docker Engine is unavailable' >&2; exit 1; }
@@ -116,18 +116,114 @@ backup_database() {
local source_sha=$1
local temp_backup=$BACKUP_DIR/.${source_sha}.dump.tmp
local final_backup=$BACKUP_DIR/${source_sha}.dump
local checksum_file=$BACKUP_DIR/${source_sha}.dump.sha256
local exclusions_file=$BACKUP_DIR/${source_sha}.excluded-tables
local restore_database=easyai_restore_${source_sha:0:12}_$$
local source_counts restored_counts
local backup_started_at backup_seconds backup_bytes restore_started_at restore_seconds
local restore_failed=false
local count_sql="SELECT (SELECT count(*) FROM gateway_wallet_transactions)::text || ':' || (SELECT count(*) FROM gateway_upload_assets)::text || ':' || (SELECT count(*) FROM gateway_cloned_voices)::text;"
local task_count_sql='SELECT (SELECT count(*) FROM gateway_tasks) + (SELECT count(*) FROM gateway_task_attempts) + (SELECT count(*) FROM gateway_task_events);'
local -a excluded_tables=(
gateway_tasks
gateway_task_attempts
gateway_task_events
gateway_task_callback_outbox
gateway_task_param_preprocessing_logs
gateway_task_message_refs
gateway_concurrency_leases
gateway_rate_limit_reservations
gateway_response_chains
settlement_outbox
river_job
)
local -a exclude_args=()
local table
for table in "${excluded_tables[@]}"; do
exclude_args+=("--exclude-table-data=$table")
done
umask 077
backup_started_at=$(date +%s)
"${compose[@]}" exec -T "$POSTGRES_SERVICE" \
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc >"$temp_backup"
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -Fc "${exclude_args[@]}" >"$temp_backup"
backup_seconds=$(( $(date +%s) - backup_started_at ))
backup_bytes=$(wc -c <"$temp_backup")
[[ -s $temp_backup ]] || { echo 'database backup is empty' >&2; rm -f "$temp_backup"; return 1; }
if (( backup_bytes > 1073741824 || backup_seconds > 300 )); then
echo "database backup control target exceeded: bytes=$backup_bytes seconds=$backup_seconds" >&2
rm -f "$temp_backup"
return 1
fi
"${compose[@]}" exec -T "$POSTGRES_SERVICE" pg_restore -l <"$temp_backup" >/dev/null
restore_started_at=$(date +%s)
"${compose[@]}" exec -T "$POSTGRES_SERVICE" \
dropdb -U "$POSTGRES_USER" --if-exists "$restore_database"
"${compose[@]}" exec -T "$POSTGRES_SERVICE" \
createdb -U "$POSTGRES_USER" "$restore_database"
if ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=pre-data <"$temp_backup"; then
restore_failed=true
fi
if [[ $restore_failed == false ]] && ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=data <"$temp_backup"; then
restore_failed=true
fi
if [[ $restore_failed == false ]]; then
if ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
psql -U "$POSTGRES_USER" -d "$restore_database" -v ON_ERROR_STOP=1 -c \
'UPDATE gateway_upload_assets SET task_id = NULL; UPDATE gateway_cloned_voices SET source_task_id = NULL, source_attempt_id = NULL;'; then
restore_failed=true
fi
fi
if [[ $restore_failed == false ]] && ! "${compose[@]}" exec -T "$POSTGRES_SERVICE" \
pg_restore -U "$POSTGRES_USER" -d "$restore_database" --section=post-data <"$temp_backup"; then
restore_failed=true
fi
if [[ $restore_failed == false ]]; then
if ! source_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
psql -U "$POSTGRES_USER" -d "$POSTGRES_DATABASE" -At -v ON_ERROR_STOP=1 -c \
"$count_sql"); then
restore_failed=true
fi
if ! restored_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
psql -U "$POSTGRES_USER" -d "$restore_database" -At -v ON_ERROR_STOP=1 -c \
"$count_sql"); then
restore_failed=true
fi
[[ $source_counts == "$restored_counts" ]] || restore_failed=true
fi
if [[ $restore_failed == false ]]; then
if ! restored_counts=$("${compose[@]}" exec -T "$POSTGRES_SERVICE" \
psql -U "$POSTGRES_USER" -d "$restore_database" -At -v ON_ERROR_STOP=1 -c \
"$task_count_sql"); then
restore_failed=true
fi
[[ $restored_counts == 0 ]] || restore_failed=true
fi
"${compose[@]}" exec -T "$POSTGRES_SERVICE" \
dropdb -U "$POSTGRES_USER" --if-exists "$restore_database"
restore_seconds=$(( $(date +%s) - restore_started_at ))
if (( restore_seconds > 300 )); then
restore_failed=true
fi
if [[ $restore_failed == true ]]; then
echo "database backup restore verification failed: seconds=$restore_seconds" >&2
rm -f "$temp_backup"
return 1
fi
mv "$temp_backup" "$final_backup"
sha256sum "$final_backup" >"$checksum_file"
printf '%s\n' "${excluded_tables[@]}" >"$exclusions_file"
mapfile -t expired < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name '*.dump' -printf '%T@ %p\n' | sort -rn | awk -v keep="$BACKUP_RETENTION" 'NR > keep { sub(/^[^ ]+ /, ""); print }')
if [[ ${#expired[@]} -gt 0 ]]; then
rm -f -- "${expired[@]}"
for table in "${expired[@]}"; do
rm -f -- "$table" "$table.sha256" "${table%.dump}.excluded-tables"
done
fi
echo "[release] verified database backup: $final_backup"
echo "[release] verified restorable task-history-free database backup: $final_backup bytes=$backup_bytes backup_seconds=$backup_seconds restore_seconds=$restore_seconds"
}
record_current() {