test(acceptance): 区分路由尝试与上游提交

额度原子抢占失败会留下 quota_race_rotated 尝试,但不会调用上游。验收报告新增 submitted 与轮转原因,并以真实提交数校验任务去重和同优先级负载分布,避免把安全轮转误判为重复提交。\n\n验证:bash -n scripts/acceptance/provider-burst.sh;shellcheck scripts/acceptance/provider-burst.sh;使用本地三 Worker 运行数据验证 25 次路由尝试中仅 24 次提交、重复远端任务和账务均为 0。
This commit is contained in:
2026-08-03 18:21:50 +08:00
parent c8d04ca731
commit eccfe568ad
+29 -11
View File
@@ -341,11 +341,26 @@ WITH configured AS (
FROM events FROM events
), peaks AS ( ), peaks AS (
SELECT platform_model_id, COALESCE(max(active),0) peak FROM points GROUP BY platform_model_id SELECT platform_model_id, COALESCE(max(active),0) peak FROM points GROUP BY platform_model_id
), attempts AS ( ), attempt_base AS (
SELECT attempt.platform_model_id, count(*) attempts SELECT attempt.platform_model_id,
attempt.remote_task_id,
COALESCE(NULLIF(attempt.metrics->>'selectionReason',''),'unspecified') selection_reason
FROM gateway_task_attempts attempt FROM gateway_task_attempts attempt
WHERE attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) WHERE attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
GROUP BY attempt.platform_model_id ), attempts AS (
SELECT platform_model_id,
count(*) attempts,
count(*) FILTER (WHERE remote_task_id IS NOT NULL) submitted
FROM attempt_base
GROUP BY platform_model_id
), attempt_reason_counts AS (
SELECT platform_model_id,selection_reason,count(*) count
FROM attempt_base
GROUP BY platform_model_id,selection_reason
), attempt_reasons AS (
SELECT platform_model_id,jsonb_object_agg(selection_reason,count ORDER BY selection_reason) reasons
FROM attempt_reason_counts
GROUP BY platform_model_id
), rate_peaks AS ( ), rate_peaks AS (
SELECT split_part(counter.scope_key, ':', 3)::uuid platform_model_id, SELECT split_part(counter.scope_key, ':', 3)::uuid platform_model_id,
max(counter.limit_value) FILTER (WHERE counter.metric='rpm') rpm_limit, max(counter.limit_value) FILTER (WHERE counter.metric='rpm') rpm_limit,
@@ -368,11 +383,14 @@ SELECT COALESCE(jsonb_agg(jsonb_build_object(
'rpmPeak', COALESCE(rate_peaks.rpm_peak,0), 'rpmPeak', COALESCE(rate_peaks.rpm_peak,0),
'tpmLimit', configured.tpm_limit, 'tpmLimit', configured.tpm_limit,
'tpmPeak', COALESCE(rate_peaks.tpm_peak,0), 'tpmPeak', COALESCE(rate_peaks.tpm_peak,0),
'attempts', COALESCE(attempts.attempts,0) 'attempts', COALESCE(attempts.attempts,0),
'submitted', COALESCE(attempts.submitted,0),
'rotationReasons', COALESCE(attempt_reasons.reasons,'{}'::jsonb)
) ORDER BY configured.concurrency_limit), '[]'::jsonb) ) ORDER BY configured.concurrency_limit), '[]'::jsonb)
FROM configured FROM configured
LEFT JOIN peaks ON peaks.platform_model_id=configured.platform_model_id LEFT JOIN peaks ON peaks.platform_model_id=configured.platform_model_id
LEFT JOIN attempts ON attempts.platform_model_id=configured.platform_model_id LEFT JOIN attempts ON attempts.platform_model_id=configured.platform_model_id
LEFT JOIN attempt_reasons ON attempt_reasons.platform_model_id=configured.platform_model_id
LEFT JOIN rate_peaks ON rate_peaks.platform_model_id=configured.platform_model_id;") LEFT JOIN rate_peaks ON rate_peaks.platform_model_id=configured.platform_model_id;")
queue=$(jq -s '{ queue=$(jq -s '{
samples:length, samples:length,
@@ -469,15 +487,15 @@ WHERE callback.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id=
resource_summary=$(provider_burst_resource_summary "$resources") resource_summary=$(provider_burst_resource_summary "$resources")
if ! jq -e --argjson requests "$requests" 'length==3 and if ! jq -e --argjson requests "$requests" 'length==3 and
[.[].priority] == [100,100,200] and [.[].priority] == [100,100,200] and
([.[].attempts]|add)==$requests and all(.[]; .attempts>0) and ([.[].submitted]|add)==$requests and all(.[]; .submitted>0) and
all(.[]; .peak<=.limit and .rpmPeak==.rpmLimit and .tpmPeak==.tpmLimit) and all(.[]; .peak<=.limit and .rpmPeak==.rpmLimit and .tpmPeak==.tpmLimit) and
.[0].peak==.[0].limit and .[1].peak==.[1].limit' \ .[0].peak==.[0].limit and .[1].peak==.[1].limit' \
<<<"$platforms" >/dev/null; then passed=false; fi <<<"$platforms" >/dev/null; then passed=false; fi
if ! jq -e '.[0].priority==.[1].priority and if ! jq -e '.[0].priority==.[1].priority and
((.[0].attempts/.[0].limit)/(.[1].attempts/.[1].limit))>=0.5 and ((.[0].submitted/.[0].limit)/(.[1].submitted/.[1].limit))>=0.5 and
((.[0].attempts/.[0].limit)/(.[1].attempts/.[1].limit))<=2 and ((.[0].submitted/.[0].limit)/(.[1].submitted/.[1].limit))<=2 and
.[2].priority>.[1].priority and .[2].priority>.[1].priority and
.[0].peak==.[0].limit and .[1].peak==.[1].limit and .[2].attempts>0' \ .[0].peak==.[0].limit and .[1].peak==.[1].limit and .[2].submitted>0' \
<<<"$platforms" >/dev/null; then passed=false; fi <<<"$platforms" >/dev/null; then passed=false; fi
if ! jq -e '.queueObserved and .admissionWaitObserved and .drainedAtEnd and .maxQueuedTasks > 0 and .maxWaitingAdmissions > 0' <<<"$queue" >/dev/null; then passed=false; fi if ! jq -e '.queueObserved and .admissionWaitObserved and .drainedAtEnd and .maxQueuedTasks > 0 and .maxWaitingAdmissions > 0' <<<"$queue" >/dev/null; then passed=false; fi
if ! jq -e --argjson requests "$requests" 'length==3 and all(.[]; .tasks>0 and .workerInstanceId!="unclaimed") and ([.[].tasks]|add)==$requests' <<<"$distribution" >/dev/null; then passed=false; fi if ! jq -e --argjson requests "$requests" 'length==3 and all(.[]; .tasks>0 and .workerInstanceId!="unclaimed") and ([.[].tasks]|add)==$requests' <<<"$distribution" >/dev/null; then passed=false; fi
@@ -507,8 +525,8 @@ WHERE callback.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id=
platforms:$platforms, platforms:$platforms,
totalConfiguredConcurrency:([$platforms[].limit]|add), totalConfiguredConcurrency:([$platforms[].limit]|add),
samePriorityLoadBalanced:($platforms[0].priority==$platforms[1].priority and samePriorityLoadBalanced:($platforms[0].priority==$platforms[1].priority and
(($platforms[0].attempts/$platforms[0].limit)/($platforms[1].attempts/$platforms[1].limit))>=0.5 and (($platforms[0].submitted/$platforms[0].limit)/($platforms[1].submitted/$platforms[1].limit))>=0.5 and
(($platforms[0].attempts/$platforms[0].limit)/($platforms[1].attempts/$platforms[1].limit))<=2), (($platforms[0].submitted/$platforms[0].limit)/($platforms[1].submitted/$platforms[1].limit))<=2),
highPriorityHardLimitsSaturated:($platforms[0].peak==$platforms[0].limit and highPriorityHardLimitsSaturated:($platforms[0].peak==$platforms[0].limit and
$platforms[1].peak==$platforms[1].limit and $platforms[1].peak==$platforms[1].limit and
$platforms[0].rpmPeak==$platforms[0].rpmLimit and $platforms[0].rpmPeak==$platforms[0].rpmLimit and
@@ -518,7 +536,7 @@ WHERE callback.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id=
lowerPrioritySpilloverObserved:($platforms[2].priority>$platforms[1].priority and lowerPrioritySpilloverObserved:($platforms[2].priority>$platforms[1].priority and
$platforms[0].peak==$platforms[0].limit and $platforms[0].peak==$platforms[0].limit and
$platforms[1].peak==$platforms[1].limit and $platforms[1].peak==$platforms[1].limit and
$platforms[2].attempts>0) $platforms[2].submitted>0)
}, },
queue:$queue, queue:$queue,
cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution, cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,