wangbo
c28bf74230
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
516 lines
25 KiB
Bash
516 lines
25 KiB
Bash
#!/usr/bin/env bash
|
|
|
|
# This library is sourced by run-local-acceptance.sh and intentionally uses its
|
|
# validated runtime globals and lifecycle state.
|
|
# shellcheck disable=SC2034,SC2154
|
|
|
|
provider_burst_platform_ids=(
|
|
f1000000-0000-4000-8000-000000000002
|
|
f1000000-0000-4000-8000-000000000004
|
|
f1000000-0000-4000-8000-000000000006
|
|
)
|
|
|
|
provider_burst_model_ids=(
|
|
f2000000-0000-4000-8000-000000000002
|
|
f2000000-0000-4000-8000-000000000004
|
|
f2000000-0000-4000-8000-000000000006
|
|
)
|
|
|
|
restore_provider_burst_platforms() {
|
|
database_query "
|
|
UPDATE integration_platforms
|
|
SET status = config->>'acceptanceBurstPreviousStatus',
|
|
config = config - 'acceptanceBurstPreviousStatus',
|
|
updated_at = statement_timestamp()
|
|
WHERE config ? 'acceptanceBurstPreviousStatus';
|
|
|
|
UPDATE integration_platforms
|
|
SET status = 'disabled',
|
|
updated_at = statement_timestamp()
|
|
WHERE COALESCE((config->>'acceptanceProviderBurst')::boolean, false);" >/dev/null
|
|
}
|
|
|
|
setup_provider_burst_platforms() {
|
|
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
|
|
[[ $acceptance_video_model =~ ^[A-Za-z0-9._:-]+$ ]]
|
|
restore_provider_burst_platforms
|
|
database_query "
|
|
WITH source AS (
|
|
SELECT platform.*
|
|
FROM integration_platforms platform
|
|
JOIN platform_models model ON model.platform_id = platform.id
|
|
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
|
WHERE base_model.invocation_name = '$acceptance_video_model'
|
|
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
|
ORDER BY platform.priority, platform.created_at
|
|
LIMIT 1
|
|
), variants(platform_id, platform_key, platform_name, concurrency_limit) AS (
|
|
VALUES
|
|
('${provider_burst_platform_ids[0]}'::uuid, 'acceptance-provider-burst-2', 'Acceptance Provider Burst C2', 2),
|
|
('${provider_burst_platform_ids[1]}'::uuid, 'acceptance-provider-burst-4', 'Acceptance Provider Burst C4', 4),
|
|
('${provider_burst_platform_ids[2]}'::uuid, 'acceptance-provider-burst-6', 'Acceptance Provider Burst C6', 6)
|
|
)
|
|
INSERT INTO integration_platforms (
|
|
id, provider, platform_key, name, base_url, auth_type, credentials, config,
|
|
visibility_scope, tenant_id, tenant_key, default_pricing_mode,
|
|
default_discount_factor, pricing_rule_set_id, retry_policy, rate_limit_policy,
|
|
priority, dynamic_priority, status, disabled_reason, cooldown_until, internal_name
|
|
)
|
|
SELECT variants.platform_id, source.provider, variants.platform_key, variants.platform_name,
|
|
source.base_url, source.auth_type, source.credentials,
|
|
source.config || jsonb_build_object(
|
|
'acceptanceProviderBurst', true,
|
|
'acceptanceBurstRunId', '$run_id',
|
|
'acceptanceConcurrencyLimit', variants.concurrency_limit
|
|
),
|
|
source.visibility_scope, source.tenant_id, source.tenant_key,
|
|
source.default_pricing_mode, source.default_discount_factor,
|
|
source.pricing_rule_set_id, source.retry_policy, '{\"rules\":[]}'::jsonb,
|
|
100, NULL, 'enabled', NULL, NULL, variants.platform_name
|
|
FROM source CROSS JOIN variants
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
provider = EXCLUDED.provider,
|
|
platform_key = EXCLUDED.platform_key,
|
|
name = EXCLUDED.name,
|
|
base_url = EXCLUDED.base_url,
|
|
auth_type = EXCLUDED.auth_type,
|
|
credentials = EXCLUDED.credentials,
|
|
config = EXCLUDED.config,
|
|
visibility_scope = EXCLUDED.visibility_scope,
|
|
tenant_id = EXCLUDED.tenant_id,
|
|
tenant_key = EXCLUDED.tenant_key,
|
|
default_pricing_mode = EXCLUDED.default_pricing_mode,
|
|
default_discount_factor = EXCLUDED.default_discount_factor,
|
|
pricing_rule_set_id = EXCLUDED.pricing_rule_set_id,
|
|
retry_policy = EXCLUDED.retry_policy,
|
|
rate_limit_policy = EXCLUDED.rate_limit_policy,
|
|
priority = EXCLUDED.priority,
|
|
dynamic_priority = EXCLUDED.dynamic_priority,
|
|
status = 'enabled',
|
|
disabled_reason = NULL,
|
|
cooldown_until = NULL,
|
|
internal_name = EXCLUDED.internal_name,
|
|
deleted_at = NULL,
|
|
updated_at = statement_timestamp();
|
|
|
|
WITH source AS (
|
|
SELECT model.*
|
|
FROM platform_models model
|
|
JOIN integration_platforms platform ON platform.id = model.platform_id
|
|
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
|
WHERE base_model.invocation_name = '$acceptance_video_model'
|
|
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
|
ORDER BY platform.priority, platform.created_at
|
|
LIMIT 1
|
|
), variants(model_id, platform_id, concurrency_limit) AS (
|
|
VALUES
|
|
('${provider_burst_model_ids[0]}'::uuid, '${provider_burst_platform_ids[0]}'::uuid, 2),
|
|
('${provider_burst_model_ids[1]}'::uuid, '${provider_burst_platform_ids[1]}'::uuid, 4),
|
|
('${provider_burst_model_ids[2]}'::uuid, '${provider_burst_platform_ids[2]}'::uuid, 6)
|
|
)
|
|
INSERT INTO platform_models (
|
|
id, platform_id, base_model_id, model_name, model_alias, model_type,
|
|
display_name, capability_override, capabilities, pricing_mode,
|
|
discount_factor, pricing_rule_set_id, billing_config_override, billing_config,
|
|
permission_config, retry_policy, rate_limit_policy, rate_limit_policy_mode,
|
|
runtime_policy_set_id, runtime_policy_override, provider_model_name,
|
|
cooldown_until, enabled
|
|
)
|
|
SELECT variants.model_id, variants.platform_id, source.base_model_id,
|
|
source.model_name, source.model_alias, source.model_type, source.display_name,
|
|
source.capability_override, source.capabilities, source.pricing_mode,
|
|
source.discount_factor, source.pricing_rule_set_id,
|
|
source.billing_config_override, source.billing_config,
|
|
source.permission_config, source.retry_policy,
|
|
jsonb_build_object('rules', jsonb_build_array(
|
|
jsonb_build_object(
|
|
'metric', 'concurrent', 'limit', variants.concurrency_limit,
|
|
'leaseTtlSeconds', 120
|
|
),
|
|
jsonb_build_object('metric', 'rpm', 'limit', 600, 'windowSeconds', 60)
|
|
)),
|
|
'override', NULL, '{}'::jsonb, source.provider_model_name, NULL, true
|
|
FROM source CROSS JOIN variants
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
platform_id = EXCLUDED.platform_id,
|
|
base_model_id = EXCLUDED.base_model_id,
|
|
model_name = EXCLUDED.model_name,
|
|
model_alias = EXCLUDED.model_alias,
|
|
model_type = EXCLUDED.model_type,
|
|
display_name = EXCLUDED.display_name,
|
|
capability_override = EXCLUDED.capability_override,
|
|
capabilities = EXCLUDED.capabilities,
|
|
pricing_mode = EXCLUDED.pricing_mode,
|
|
discount_factor = EXCLUDED.discount_factor,
|
|
pricing_rule_set_id = EXCLUDED.pricing_rule_set_id,
|
|
billing_config_override = EXCLUDED.billing_config_override,
|
|
billing_config = EXCLUDED.billing_config,
|
|
permission_config = EXCLUDED.permission_config,
|
|
retry_policy = EXCLUDED.retry_policy,
|
|
rate_limit_policy = EXCLUDED.rate_limit_policy,
|
|
rate_limit_policy_mode = 'override',
|
|
runtime_policy_set_id = NULL,
|
|
runtime_policy_override = '{}'::jsonb,
|
|
provider_model_name = EXCLUDED.provider_model_name,
|
|
cooldown_until = NULL,
|
|
enabled = true,
|
|
updated_at = statement_timestamp();
|
|
|
|
WITH selected_platforms AS (
|
|
SELECT platform.id
|
|
FROM integration_platforms platform
|
|
JOIN platform_models model ON model.platform_id = platform.id
|
|
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
|
WHERE base_model.invocation_name = '$acceptance_video_model'
|
|
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
|
AND NOT COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
)
|
|
UPDATE integration_platforms platform
|
|
SET config = jsonb_set(
|
|
platform.config,
|
|
'{acceptanceBurstPreviousStatus}',
|
|
to_jsonb(platform.status),
|
|
true
|
|
),
|
|
status = 'disabled',
|
|
updated_at = statement_timestamp()
|
|
FROM selected_platforms
|
|
WHERE platform.id = selected_platforms.id;
|
|
|
|
WITH acceptance_group AS (
|
|
SELECT api_key.user_group_id AS id
|
|
FROM gateway_acceptance_runs run
|
|
JOIN gateway_api_keys api_key ON api_key.id::text = run.api_key_id
|
|
WHERE run.id = '$run_id'::uuid
|
|
), resources(resource_type, resource_id) AS (
|
|
SELECT 'platform'::text, platform.id
|
|
FROM integration_platforms platform
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
|
UNION ALL
|
|
SELECT 'platform_model', model.id
|
|
FROM platform_models model
|
|
JOIN integration_platforms platform ON platform.id = model.platform_id
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
|
UNION
|
|
SELECT 'base_model', model.base_model_id
|
|
FROM platform_models model
|
|
JOIN integration_platforms platform ON platform.id = model.platform_id
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
|
)
|
|
INSERT INTO gateway_access_rules (
|
|
subject_type, subject_id, resource_type, resource_id, effect,
|
|
priority, min_permission_level, conditions, metadata, status
|
|
)
|
|
SELECT 'user_group', acceptance_group.id, resources.resource_type,
|
|
resources.resource_id, 'allow', 1, 0, '{}'::jsonb,
|
|
jsonb_build_object('purpose', 'provider_burst_acceptance', 'runId', '$run_id'),
|
|
'active'
|
|
FROM acceptance_group CROSS JOIN resources
|
|
WHERE acceptance_group.id IS NOT NULL
|
|
ON CONFLICT (subject_type, subject_id, resource_type, resource_id, effect)
|
|
DO UPDATE SET status = 'active', metadata = EXCLUDED.metadata,
|
|
updated_at = statement_timestamp();" >/dev/null
|
|
|
|
local configured
|
|
configured=$(database_query "
|
|
SELECT jsonb_build_object(
|
|
'platforms', count(*),
|
|
'limits', jsonb_agg((platform.config->>'acceptanceConcurrencyLimit')::int ORDER BY (platform.config->>'acceptanceConcurrencyLimit')::int),
|
|
'enabled', count(*) FILTER (WHERE platform.status='enabled' AND model.enabled)
|
|
)
|
|
FROM integration_platforms platform
|
|
JOIN platform_models model ON model.platform_id = platform.id
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId' = '$run_id';")
|
|
jq -e '.platforms == 3 and .enabled == 3 and .limits == [2,4,6]' <<<"$configured" >/dev/null
|
|
}
|
|
|
|
sample_provider_burst() {
|
|
local output=$1 stop_file=$2
|
|
: >"$output"
|
|
while [[ ! -e $stop_file ]]; do
|
|
database_query "
|
|
WITH configured AS (
|
|
SELECT platform.id platform_id, platform.platform_key, model.id platform_model_id,
|
|
(platform.config->>'acceptanceConcurrencyLimit')::int concurrency_limit
|
|
FROM integration_platforms platform
|
|
JOIN platform_models model ON model.platform_id = platform.id
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
|
), platform_state AS (
|
|
SELECT configured.*,
|
|
(SELECT count(*) FROM gateway_concurrency_leases lease
|
|
WHERE lease.scope_type='platform_model'
|
|
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
|
AND lease.released_at IS NULL
|
|
AND lease.expires_at > statement_timestamp()) active_leases,
|
|
(SELECT count(*) FROM gateway_task_attempts attempt
|
|
WHERE attempt.platform_model_id=configured.platform_model_id
|
|
AND attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)) attempts
|
|
FROM configured
|
|
)
|
|
SELECT jsonb_build_object(
|
|
'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'),
|
|
'queuedTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='queued'),
|
|
'runningTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='running'),
|
|
'succeededTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='succeeded'),
|
|
'waitingAdmissions', (SELECT count(*) FROM gateway_task_admissions admission JOIN gateway_tasks task ON task.id=admission.task_id WHERE task.acceptance_run_id='$run_id'::uuid AND admission.status='waiting'),
|
|
'platforms', COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
|
'platformId', platform_id,
|
|
'platformModelId', platform_model_id,
|
|
'platformKey', platform_key,
|
|
'limit', concurrency_limit,
|
|
'activeLeases', active_leases,
|
|
'attempts', attempts
|
|
) ORDER BY concurrency_limit) FROM platform_state), '[]'::jsonb)
|
|
);" >>"$output" || true
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
provider_burst_resource_summary() {
|
|
local resources=$1
|
|
jq -Rn '
|
|
def number_without($suffix): rtrimstr($suffix) | tonumber;
|
|
def memory_mib:
|
|
if endswith("Gi") then number_without("Gi") * 1024
|
|
elif endswith("Mi") then number_without("Mi")
|
|
elif endswith("Ki") then number_without("Ki") / 1024
|
|
else tonumber end;
|
|
[inputs | split(",") | select(.[0] != "timestamp") | {
|
|
scope:.[1],name:.[2],cpu:.[3],memory:.[4]
|
|
}]
|
|
| group_by([.scope,.name])
|
|
| map(if .[0].scope == "node" then {
|
|
scope:"node",name:.[0].name,
|
|
maxCpuPercent:(map(.cpu | number_without("%")) | max),
|
|
maxMemoryPercent:(map(.memory | number_without("%")) | max)
|
|
} else {
|
|
scope:"pod",name:.[0].name,
|
|
maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max),
|
|
maxMemoryMiB:(map(.memory | memory_mib) | max)
|
|
} end)' <"$resources"
|
|
}
|
|
|
|
build_provider_burst_report() {
|
|
local output=$1 load_report=$2 samples=$3 worker_samples=$4 resources=$5 requests=$6
|
|
local platforms queue workers distribution worker_peaks limits concurrency_peaks
|
|
local leaks duplicates callbacks resource_summary passed=true
|
|
platforms=$(database_query "
|
|
WITH configured AS (
|
|
SELECT platform.id platform_id, platform.platform_key, model.id platform_model_id,
|
|
(platform.config->>'acceptanceConcurrencyLimit')::int concurrency_limit
|
|
FROM integration_platforms platform
|
|
JOIN platform_models model ON model.platform_id=platform.id
|
|
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
|
AND platform.config->>'acceptanceBurstRunId'='$run_id'
|
|
), events AS (
|
|
SELECT configured.platform_model_id, lease.acquired_at event_at, lease.lease_value delta
|
|
FROM configured
|
|
JOIN gateway_concurrency_leases lease
|
|
ON lease.scope_type='platform_model'
|
|
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
|
UNION ALL
|
|
SELECT configured.platform_model_id,
|
|
LEAST(COALESCE(lease.released_at, lease.expires_at), lease.expires_at),
|
|
-lease.lease_value
|
|
FROM configured
|
|
JOIN gateway_concurrency_leases lease
|
|
ON lease.scope_type='platform_model'
|
|
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
|
), points AS (
|
|
SELECT platform_model_id,
|
|
SUM(delta) OVER (PARTITION BY platform_model_id ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
|
FROM events
|
|
), peaks AS (
|
|
SELECT platform_model_id, COALESCE(max(active),0) peak FROM points GROUP BY platform_model_id
|
|
), attempts AS (
|
|
SELECT attempt.platform_model_id, count(*) attempts
|
|
FROM gateway_task_attempts attempt
|
|
WHERE attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
|
GROUP BY attempt.platform_model_id
|
|
)
|
|
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
|
'platformId', configured.platform_id,
|
|
'platformModelId', configured.platform_model_id,
|
|
'platformKey', configured.platform_key,
|
|
'limit', configured.concurrency_limit,
|
|
'peak', COALESCE(peaks.peak,0),
|
|
'attempts', COALESCE(attempts.attempts,0)
|
|
) ORDER BY configured.concurrency_limit), '[]'::jsonb)
|
|
FROM configured
|
|
LEFT JOIN peaks ON peaks.platform_model_id=configured.platform_model_id
|
|
LEFT JOIN attempts ON attempts.platform_model_id=configured.platform_model_id;")
|
|
queue=$(jq -s '{
|
|
samples:length,
|
|
maxQueuedTasks:(map(.queuedTasks)|max),
|
|
maxRunningTasks:(map(.runningTasks)|max),
|
|
maxWaitingAdmissions:(map(.waitingAdmissions)|max),
|
|
queueObserved:any(.queuedTasks > 0),
|
|
admissionWaitObserved:any(.waitingAdmissions > 0),
|
|
drainedAtEnd:(.[-1].queuedTasks == 0 and .[-1].runningTasks == 0 and .[-1].waitingAdmissions == 0)
|
|
}' "$samples")
|
|
workers=$(database_query "
|
|
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
|
'instanceId',instance_id,'podName',pod_name,'site',site,
|
|
'allocatedCapacity',allocated_capacity,'safeCapacity',safe_capacity,
|
|
'heavyCapacity',heavy_capacity,'pressureState',pressure_state
|
|
) ORDER BY site,instance_id),'[]'::jsonb)
|
|
FROM gateway_worker_instances
|
|
WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';")
|
|
distribution=$(database_query "
|
|
SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb)
|
|
FROM (
|
|
SELECT COALESCE(
|
|
regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''),
|
|
'unclaimed'
|
|
) worker_id,
|
|
count(*) tasks
|
|
FROM gateway_tasks task
|
|
LEFT JOIN river_job job ON job.id=task.river_job_id
|
|
WHERE task.acceptance_run_id='$run_id'::uuid
|
|
GROUP BY worker_id
|
|
) grouped;")
|
|
worker_peaks=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({
|
|
instanceId:.[0].instanceId,
|
|
peakActiveTasks:(map(.activeTasks)|max),
|
|
peakPreparingTasks:(map(.preparingTasks)|max),
|
|
peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max),
|
|
peakFinalizingTasks:(map(.finalizingTasks)|max),
|
|
minSafeCapacity:(map(.safeCapacity)|min),
|
|
maxSafeCapacity:(map(.safeCapacity)|max),
|
|
pressureStates:(map(.pressureState)|unique)
|
|
})' "$worker_samples")
|
|
limits=$(database_query "
|
|
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
|
'scopeType',scope_type,'scopeKey',scope_key,'metric',metric,
|
|
'limit',limit_value,'used',used_value,'reserved',reserved_value,
|
|
'windowStart',window_start,'resetAt',reset_at
|
|
) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb)
|
|
FROM gateway_rate_limit_counters
|
|
WHERE scope_key LIKE 'acceptance:$run_id:%';")
|
|
concurrency_peaks=$(database_query "
|
|
WITH events AS (
|
|
SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta
|
|
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
|
UNION ALL
|
|
SELECT scope_type,scope_key,limit_value,
|
|
LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value
|
|
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
|
), points AS (
|
|
SELECT scope_type,scope_key,limit_value,
|
|
SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
|
FROM events
|
|
), peaks AS (
|
|
SELECT scope_type,scope_key,max(limit_value) limit_value,COALESCE(max(active),0) peak
|
|
FROM points GROUP BY scope_type,scope_key
|
|
)
|
|
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
|
'scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak
|
|
) ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;")
|
|
leaks=$(database_query "
|
|
SELECT jsonb_build_object(
|
|
'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>statement_timestamp()),
|
|
'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'),
|
|
'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled'))
|
|
);")
|
|
duplicates=$(database_query "
|
|
SELECT jsonb_build_object(
|
|
'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate),
|
|
'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate)
|
|
);")
|
|
callbacks=$(database_query "
|
|
SELECT jsonb_build_object(
|
|
'deliveries',count(*) FILTER (WHERE callback.status='delivered'),
|
|
'pending',count(*) FILTER (WHERE callback.status<>'delivered'),
|
|
'duplicates',(SELECT count(*) FROM (
|
|
SELECT callback_inner.task_id,callback_inner.seq,callback_inner.callback_url
|
|
FROM gateway_task_callback_outbox callback_inner
|
|
WHERE callback_inner.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
|
GROUP BY callback_inner.task_id,callback_inner.seq,callback_inner.callback_url
|
|
HAVING count(*)>1
|
|
) duplicate)
|
|
)
|
|
FROM gateway_task_callback_outbox callback
|
|
WHERE callback.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid);")
|
|
resource_summary=$(provider_burst_resource_summary "$resources")
|
|
if ! jq -e 'length==3 and all(.[]; .peak == .limit and .attempts > 0)' <<<"$platforms" >/dev/null; then passed=false; fi
|
|
if ! jq -e '.queueObserved and .admissionWaitObserved and .drainedAtEnd and .maxQueuedTasks > 0 and .maxWaitingAdmissions > 0' <<<"$queue" >/dev/null; then passed=false; fi
|
|
if ! jq -e --argjson requests "$requests" 'length==3 and all(.[]; .tasks>0 and .workerInstanceId!="unclaimed") and ([.[].tasks]|add)==$requests' <<<"$distribution" >/dev/null; then passed=false; fi
|
|
if ! jq -e 'length==3 and all(.[]; .peakActiveTasks>0 and (.pressureStates|index("critical")|not))' <<<"$worker_peaks" >/dev/null; then passed=false; fi
|
|
if ! jq -e 'length>0 and all(.[]; (.used+.reserved)<=.limit)' <<<"$limits" >/dev/null; then passed=false; fi
|
|
if ! jq -e 'all(.[]; .peak<=.limit)' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi
|
|
if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi
|
|
if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi
|
|
if ! jq -e '.deliveries>0 and .pending==0 and .duplicates==0' <<<"$callbacks" >/dev/null; then passed=false; fi
|
|
if ! jq -e --argjson requests "$requests" '.passed and ([.phases[].completed]|add)==$requests and ([.phases[].failed]|add)==0' "$load_report" >/dev/null; then passed=false; fi
|
|
jq -n \
|
|
--arg runId "$run_id" --arg model "$acceptance_video_model" \
|
|
--argjson passed "$passed" --argjson requests "$requests" \
|
|
--argjson phases "$(jq '.phases' "$load_report")" \
|
|
--argjson platforms "$platforms" --argjson queue "$queue" \
|
|
--argjson workers "$workers" --argjson distribution "$distribution" \
|
|
--argjson workerPeaks "$worker_peaks" --argjson rateLimits "$limits" \
|
|
--argjson concurrencyPeaks "$concurrency_peaks" \
|
|
--argjson resources "$resource_summary" --argjson leaks "$leaks" \
|
|
--argjson duplicates "$duplicates" --argjson callbacks "$callbacks" \
|
|
'{
|
|
schemaVersion:"acceptance-provider-burst-report/v1",
|
|
runId:$runId,profile:"provider-burst",model:$model,
|
|
startedAt:(now|todateiso8601),finishedAt:(now|todateiso8601),
|
|
passed:$passed,secretSafe:true,requests:$requests,phases:$phases,
|
|
providerRouting:{platforms:$platforms,totalConfiguredConcurrency:([$platforms[].limit]|add)},
|
|
queue:$queue,
|
|
cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,
|
|
rateLimits:$rateLimits,concurrencyPeaks:$concurrencyPeaks,
|
|
resourcePeaks:$resources,leaks:$leaks,duplicates:$duplicates,callbacks:$callbacks}
|
|
}' >"$output"
|
|
chmod 0600 "$output"
|
|
jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null
|
|
}
|
|
|
|
provider_burst() {
|
|
local requests=${AI_GATEWAY_LOCAL_PROVIDER_BURST_REQUESTS:-48}
|
|
local load_report="$report_root/provider-burst-load.json"
|
|
local samples="$report_root/provider-burst-samples.ndjson"
|
|
local worker_samples="$report_root/provider-burst-worker-samples.ndjson"
|
|
local resources="$report_root/provider-burst-resources.csv"
|
|
local report="$report_root/provider-burst.json"
|
|
worker_sample_stop="$report_root/provider-burst-samples.stop"
|
|
[[ $requests =~ ^[0-9]+$ ]] && ((requests >= 24 && requests <= 256))
|
|
[[ ! -e $load_report && ! -e $samples && ! -e $worker_samples && ! -e $resources && ! -e $report && ! -e $worker_sample_stop ]] ||
|
|
fail_gate acceptance_report_exists "provider burst evidence already exists"
|
|
current_phase=provider_burst_platforms
|
|
setup_provider_burst_platforms
|
|
provider_burst_configured=true
|
|
current_phase=provider_burst_workers
|
|
configure_adaptive_workers
|
|
sample_provider_burst "$samples" "$worker_sample_stop" &
|
|
active_provider_sampler_pid=$!
|
|
sample_worker_runtime "$worker_samples" "$worker_sample_stop" &
|
|
active_worker_sampler_pid=$!
|
|
sample_resources "$resources" "$worker_sample_stop" &
|
|
active_resource_sampler_pid=$!
|
|
current_phase=provider_burst_load
|
|
run_load video-throughput "$load_report" -requests "$requests"
|
|
sleep 2
|
|
touch "$worker_sample_stop"
|
|
wait "$active_provider_sampler_pid"
|
|
wait "$active_worker_sampler_pid"
|
|
wait "$active_resource_sampler_pid"
|
|
active_provider_sampler_pid=
|
|
active_worker_sampler_pid=
|
|
active_resource_sampler_pid=
|
|
worker_sample_stop=
|
|
current_phase=provider_burst_hard_gates
|
|
verify_hard_gates
|
|
current_phase=provider_burst_report
|
|
build_provider_burst_report "$report" "$load_report" "$samples" "$worker_samples" "$resources" "$requests"
|
|
restore_provider_burst_platforms
|
|
provider_burst_configured=false
|
|
echo "local_acceptance_provider_burst=PASS run_id=$run_id requests=$requests report=$report"
|
|
}
|