feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -13,20 +13,32 @@ snapshot="$state_root/snapshot.json"
|
||||
load_binary="$state_root/easyai-ai-gateway-acceptance-load"
|
||||
identity_file="$state_root/control-plane-identity.json"
|
||||
active_load_pid=
|
||||
active_worker_sampler_pid=
|
||||
active_resource_sampler_pid=
|
||||
active_provider_sampler_pid=
|
||||
worker_sample_stop=
|
||||
netem_active=false
|
||||
provider_burst_configured=false
|
||||
current_phase=initialization
|
||||
stable_profile=
|
||||
failure_gate_id=local_execution_incomplete
|
||||
# shellcheck source=scripts/acceptance/local-control-plane.sh
|
||||
source "$script_dir/local-control-plane.sh"
|
||||
# shellcheck source=scripts/acceptance/provider-burst.sh
|
||||
source "$script_dir/provider-burst.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/acceptance/run-local-acceptance.sh quick
|
||||
scripts/acceptance/run-local-acceptance.sh adaptive
|
||||
scripts/acceptance/run-local-acceptance.sh provider-burst
|
||||
scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/<SHA>.json
|
||||
scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/<SHA>.json
|
||||
|
||||
`adaptive` runs a three-Worker emulator-only concurrency ladder without changing
|
||||
provider limits. `provider-burst` binds one video model to three emulator-only
|
||||
platforms with concurrency limits 2/4/6 and validates burst queueing and routing.
|
||||
`full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain,
|
||||
80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process
|
||||
runs outside K3s and splits requests 50/50 across both TLS entrances.
|
||||
@@ -51,9 +63,24 @@ cleanup() {
|
||||
kill "$active_load_pid" >/dev/null 2>&1 || true
|
||||
wait "$active_load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_worker_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_worker_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_resource_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_resource_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_provider_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_provider_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $netem_active == true ]]; then
|
||||
"$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $provider_burst_configured == true && -n ${runtime:-} ]]; then
|
||||
restore_provider_burst_platforms >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
|
||||
restore_profile_best_effort "${stable_profile:-P24}"
|
||||
if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
|
||||
@@ -206,10 +233,281 @@ run_load() {
|
||||
return 1
|
||||
fi
|
||||
((load_status == 0)) || return "$load_status"
|
||||
jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
|
||||
jq -e '.schemaVersion == "acceptance-load-report/v2" and .secretSafe == true and .passed == true' \
|
||||
"$report_path" >/dev/null
|
||||
}
|
||||
|
||||
sample_worker_runtime() {
|
||||
local output=$1 stop_file=$2
|
||||
: >"$output"
|
||||
while [[ ! -e $stop_file ]]; do
|
||||
database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'),
|
||||
'workers', COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instanceId', instance_id,
|
||||
'allocatedCapacity', allocated_capacity,
|
||||
'safeCapacity', safe_capacity,
|
||||
'heavyCapacity', heavy_capacity,
|
||||
'activeTasks', active_tasks,
|
||||
'preparingTasks', preparing_tasks,
|
||||
'waitingUpstreamTasks', waiting_upstream_tasks,
|
||||
'finalizingTasks', finalizing_tasks,
|
||||
'pressureState', pressure_state,
|
||||
'pressureReason', pressure_reason
|
||||
) ORDER BY instance_id), '[]'::jsonb)
|
||||
)
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';" >>"$output" || true
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_adaptive_workloads_settled() {
|
||||
local deadline=$((SECONDS + 180)) stable_samples=0 pod_state
|
||||
local api_total api_ready api_terminating worker_total worker_ready worker_terminating
|
||||
while ((SECONDS < deadline)); do
|
||||
pod_state=$(kubectl --context "$context" -n "$namespace" get pods \
|
||||
-l 'app.kubernetes.io/name in (easyai-api,easyai-worker)' -o json) || {
|
||||
stable_samples=0
|
||||
sleep 2
|
||||
continue
|
||||
}
|
||||
read -r api_total api_ready api_terminating worker_total worker_ready worker_terminating < <(
|
||||
jq -r '
|
||||
def total($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp == null)] | length;
|
||||
def ready($name): [.items[] | select(
|
||||
.metadata.labels["app.kubernetes.io/name"] == $name and
|
||||
.metadata.deletionTimestamp == null and
|
||||
.status.phase == "Running" and
|
||||
((.status.containerStatuses // []) | length) > 0 and
|
||||
all(.status.containerStatuses[]; .ready == true)
|
||||
)] | length;
|
||||
def terminating($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp != null)] | length;
|
||||
[total("easyai-api"), ready("easyai-api"), terminating("easyai-api"), total("easyai-worker"), ready("easyai-worker"), terminating("easyai-worker")] | @tsv
|
||||
' <<<"$pod_state"
|
||||
)
|
||||
if [[ $api_total == 2 && $api_ready == 2 && $api_terminating == 0 &&
|
||||
$worker_total == 3 && $worker_ready == 3 && $worker_terminating == 0 ]]; then
|
||||
stable_samples=$((stable_samples + 1))
|
||||
if ((stable_samples >= 3)); then
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
stable_samples=0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
fail_gate adaptive_workload_settle_timeout \
|
||||
"adaptive workloads did not settle at two Ready APIs and three Ready Workers without terminating Pods"
|
||||
}
|
||||
|
||||
configure_adaptive_workers() {
|
||||
local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
|
||||
local global_limit=$((hard_limit * 3))
|
||||
[[ $hard_limit =~ ^[0-9]+$ ]] && ((hard_limit >= 4 && hard_limit <= 256))
|
||||
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
||||
--type=merge -p "$(jq -cn --arg hard "$hard_limit" --arg global "$global_limit" '{data:{
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
||||
AI_GATEWAY_WORKER_LOAD_MODE:"adaptive",
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$hard,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$global,
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
|
||||
}}')" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout restart \
|
||||
deployment/easyai-capacity-controller >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status \
|
||||
deployment/easyai-capacity-controller --timeout=10m
|
||||
kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-ningbo --replicas=2 >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-hongkong --replicas=1 >/dev/null
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
||||
AI_GATEWAY_WORKER_LOAD_MODE=adaptive \
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$hard_limit" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
wait_for_adaptive_workloads_settled
|
||||
local ready
|
||||
ready=$(kubectl --context "$context" -n "$namespace" get pods -l app.kubernetes.io/name=easyai-worker -o json |
|
||||
jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
|
||||
[[ $ready == 3 ]] || fail_gate adaptive_worker_count "adaptive validation requires exactly three Ready Workers"
|
||||
}
|
||||
|
||||
run_adaptive_level() {
|
||||
local concurrency=$1 passed=true etcd_healthy=true
|
||||
local gemini_report="$report_root/adaptive-load-${concurrency}-gemini.json"
|
||||
local video_report="$report_root/adaptive-load-${concurrency}-video.json"
|
||||
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
||||
fail_gate local_control_plane_restarted "local K3s server identity changed before adaptive level $concurrency"
|
||||
record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
||||
fail_gate local_control_plane_identity "failed to record adaptive level $concurrency control-plane baseline"
|
||||
if ! run_load gemini-multi-image "$gemini_report" -requests "$concurrency"; then
|
||||
passed=false
|
||||
fi
|
||||
if [[ $passed == true ]] && ! run_load video-throughput "$video_report" -requests "$concurrency"; then
|
||||
passed=false
|
||||
fi
|
||||
local critical
|
||||
critical=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND pressure_state='critical' AND heartbeat_at > now() - interval '30 seconds';")
|
||||
if ((critical > 0)); then
|
||||
passed=false
|
||||
fi
|
||||
if ! verify_local_etcd_runtime_logs "$cluster_name" "$identity_file"; then
|
||||
passed=false
|
||||
etcd_healthy=false
|
||||
record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || return 1
|
||||
fi
|
||||
jq -n --argjson concurrency "$concurrency" --argjson passed "$passed" \
|
||||
--argjson etcdHealthy "$etcd_healthy" \
|
||||
--arg gemini "$(basename "$gemini_report")" --arg video "$(basename "$video_report")" \
|
||||
'{concurrency:$concurrency,passed:$passed,etcdHealthy:$etcdHealthy,reports:[$gemini,$video]}' \
|
||||
>"$report_root/adaptive-level-${concurrency}.json"
|
||||
chmod 0600 "$report_root/adaptive-level-${concurrency}.json"
|
||||
[[ $passed == true ]]
|
||||
}
|
||||
|
||||
build_adaptive_report() {
|
||||
local output=$1 samples=$2 resources=$3 highest=$4 failed=${5:-0}
|
||||
local levels workers distribution routing limits concurrency_peaks leaks duplicates sample_summary resource_summary phases
|
||||
levels=$(jq -s 'sort_by(.concurrency)' "$report_root"/adaptive-level-*.json)
|
||||
workers=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instanceId',instance_id,'podName',pod_name,'site',site,'status',status,
|
||||
'allocatedCapacity',allocated_capacity,'hardCapacityLimit',hard_capacity_limit,
|
||||
'safeCapacity',safe_capacity,'heavyCapacity',heavy_capacity,
|
||||
'activeTasks',active_tasks,'preparingTasks',preparing_tasks,
|
||||
'waitingUpstreamTasks',waiting_upstream_tasks,'finalizingTasks',finalizing_tasks,
|
||||
'pressureState',pressure_state,'pressureReason',pressure_reason,
|
||||
'loadSampledAt',load_sampled_at
|
||||
) ORDER BY site,instance_id),'[]'::jsonb)
|
||||
FROM gateway_worker_instances
|
||||
WHERE heartbeat_at > now() - interval '30 seconds';")
|
||||
distribution=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb)
|
||||
FROM (
|
||||
SELECT COALESCE(
|
||||
regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''),
|
||||
'unclaimed'
|
||||
) worker_id,
|
||||
count(*) tasks
|
||||
FROM gateway_tasks task
|
||||
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE task.acceptance_run_id='$run_id'::uuid
|
||||
GROUP BY worker_id
|
||||
) grouped;")
|
||||
routing=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'platformId',platform_id,'platformModelId',platform_model_id,'attempts',attempts
|
||||
) ORDER BY platform_id,platform_model_id),'[]'::jsonb)
|
||||
FROM (SELECT platform_id::text,platform_model_id::text,count(*) attempts
|
||||
FROM gateway_task_attempts WHERE task_id IN
|
||||
(SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
||||
GROUP BY platform_id,platform_model_id) grouped;")
|
||||
limits=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'scopeType',scope_type,'scopeKey',scope_key,'metric',metric,
|
||||
'limit',limit_value,'used',used_value,'reserved',reserved_value,
|
||||
'windowStart',window_start,'resetAt',reset_at
|
||||
) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb)
|
||||
FROM gateway_rate_limit_counters
|
||||
WHERE scope_key LIKE 'acceptance:$run_id:%';")
|
||||
concurrency_peaks=$(database_query "
|
||||
WITH events AS (
|
||||
SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta FROM gateway_concurrency_leases
|
||||
WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
UNION ALL
|
||||
SELECT scope_type,scope_key,limit_value,LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value
|
||||
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
), points AS (
|
||||
SELECT scope_type,scope_key,limit_value,
|
||||
SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
||||
FROM events
|
||||
), peaks AS (
|
||||
SELECT scope_type,scope_key,MAX(limit_value) limit_value,COALESCE(MAX(active),0) peak
|
||||
FROM points GROUP BY scope_type,scope_key
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object('scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak)
|
||||
ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;")
|
||||
leaks=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>now()),
|
||||
'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'),
|
||||
'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled'))
|
||||
);")
|
||||
duplicates=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate),
|
||||
'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate)
|
||||
);")
|
||||
sample_summary=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({
|
||||
instanceId:.[0].instanceId,
|
||||
peakActiveTasks:(map(.activeTasks)|max),
|
||||
peakPreparingTasks:(map(.preparingTasks)|max),
|
||||
peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max),
|
||||
peakFinalizingTasks:(map(.finalizingTasks)|max),
|
||||
minSafeCapacity:(map(.safeCapacity)|min),
|
||||
maxSafeCapacity:(map(.safeCapacity)|max),
|
||||
pressureStates:(map(.pressureState)|unique)
|
||||
})' "$samples")
|
||||
resource_summary=$(jq -Rn '
|
||||
def number_without($suffix): rtrimstr($suffix) | tonumber;
|
||||
def memory_mib:
|
||||
if endswith("Gi") then number_without("Gi") * 1024
|
||||
elif endswith("Mi") then number_without("Mi")
|
||||
elif endswith("Ki") then number_without("Ki") / 1024
|
||||
else tonumber end;
|
||||
[inputs | split(",") | select(.[0] != "timestamp") | {
|
||||
scope:.[1],name:.[2],cpu:.[3],memory:.[4]
|
||||
}]
|
||||
| group_by([.scope,.name])
|
||||
| map(if .[0].scope == "node" then {
|
||||
scope:"node",name:.[0].name,
|
||||
maxCpuPercent:(map(.cpu | number_without("%")) | max),
|
||||
maxMemoryPercent:(map(.memory | number_without("%")) | max)
|
||||
} else {
|
||||
scope:"pod",name:.[0].name,
|
||||
maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max),
|
||||
maxMemoryMiB:(map(.memory | memory_mib) | max)
|
||||
} end)' <"$resources")
|
||||
phases=$(jq -s '[.[] | .phases[]]' "$report_root"/adaptive-load-*.json)
|
||||
local passed=true bottleneck=configured_probe_ceiling
|
||||
if ((highest < 1)); then passed=false; fi
|
||||
if ((failed > 0)); then bottleneck=worker_or_gateway_limit; fi
|
||||
if jq -e 'any(.[]; .etcdHealthy == false)' <<<"$levels" >/dev/null; then
|
||||
bottleneck=local_control_plane_limit
|
||||
fi
|
||||
if ! jq -e 'length == 3' <<<"$workers" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'all(.[]; ((.used + .reserved) <= .limit))' <<<"$limits" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'all(.[]; (.limit == null or .peak <= .limit))' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi
|
||||
jq -n \
|
||||
--arg runId "$run_id" --arg startedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
--arg bottleneck "$bottleneck" --argjson passed "$passed" \
|
||||
--argjson highest "$highest" --argjson failed "$failed" \
|
||||
--argjson levels "$levels" --argjson phases "$phases" \
|
||||
--argjson workers "$workers" --argjson distribution "$distribution" \
|
||||
--argjson routing "$routing" --argjson limits "$limits" \
|
||||
--argjson concurrencyPeaks "$concurrency_peaks" --argjson workerPeaks "$sample_summary" \
|
||||
--argjson resourcePeaks "$resource_summary" \
|
||||
--argjson leaks "$leaks" --argjson duplicates "$duplicates" \
|
||||
'{
|
||||
schemaVersion:"acceptance-load-report/v2",runId:$runId,profile:"adaptive-capacity",
|
||||
startedAt:$startedAt,finishedAt:(now|todateiso8601),passed:$passed,secretSafe:true,
|
||||
phases:$phases,
|
||||
capacity:{levels:$levels,highestStableConcurrency:$highest,firstFailedConcurrency:$failed,bottleneck:$bottleneck},
|
||||
cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,
|
||||
platformRouting:$routing,rateLimits:$limits,concurrencyPeaks:$concurrencyPeaks,
|
||||
resourcePeaks:$resourcePeaks,leaks:$leaks,duplicates:$duplicates}
|
||||
}' >"$output"
|
||||
chmod 0600 "$output"
|
||||
jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null
|
||||
}
|
||||
|
||||
sample_resources() {
|
||||
local output=$1 stop_file=$2
|
||||
printf 'timestamp,scope,name,cpu,memory\n' >"$output"
|
||||
@@ -567,6 +865,64 @@ quick() {
|
||||
echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json"
|
||||
}
|
||||
|
||||
adaptive() {
|
||||
local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
|
||||
local max_probe=${AI_GATEWAY_LOCAL_ADAPTIVE_MAX_CONCURRENCY:-$((hard_limit * 3))}
|
||||
local level=4 highest=0 failed=0 low high mid
|
||||
local samples="$report_root/adaptive-worker-samples.ndjson"
|
||||
local resources="$report_root/adaptive-resources.csv"
|
||||
worker_sample_stop="$report_root/adaptive-worker-samples.stop"
|
||||
[[ ! -e $samples && ! -e $resources && ! -e $worker_sample_stop ]] || fail_gate acceptance_report_exists "adaptive evidence already exists"
|
||||
current_phase=adaptive_workers
|
||||
configure_adaptive_workers
|
||||
sample_worker_runtime "$samples" "$worker_sample_stop" &
|
||||
active_worker_sampler_pid=$!
|
||||
sample_resources "$resources" "$worker_sample_stop" &
|
||||
active_resource_sampler_pid=$!
|
||||
while ((level <= max_probe)); do
|
||||
current_phase="adaptive_capacity_$level"
|
||||
if run_adaptive_level "$level"; then
|
||||
highest=$level
|
||||
level=$((level * 2))
|
||||
else
|
||||
failed=$level
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ((failed > 0 && highest > 0)); then
|
||||
low=$((highest + 1))
|
||||
high=$((failed - 1))
|
||||
while ((low <= high)); do
|
||||
mid=$(((low + high) / 2))
|
||||
current_phase="adaptive_capacity_refine_$mid"
|
||||
if run_adaptive_level "$mid"; then
|
||||
highest=$mid
|
||||
low=$((mid + 1))
|
||||
else
|
||||
failed=$mid
|
||||
high=$((mid - 1))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
touch "$worker_sample_stop"
|
||||
wait "$active_worker_sampler_pid"
|
||||
wait "$active_resource_sampler_pid"
|
||||
active_worker_sampler_pid=
|
||||
active_resource_sampler_pid=
|
||||
worker_sample_stop=
|
||||
local ready_workers registered_workers
|
||||
ready_workers=$(kubectl --context "$context" -n "$namespace" get pods \
|
||||
-l app.kubernetes.io/name=easyai-worker -o json |
|
||||
jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
|
||||
registered_workers=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';")
|
||||
[[ $ready_workers == 3 && $registered_workers == 3 ]] ||
|
||||
fail_gate adaptive_worker_count "adaptive validation requires three Ready and registered Workers throughout the final gate"
|
||||
verify_hard_gates
|
||||
current_phase=adaptive_report
|
||||
build_adaptive_report "$report_root/adaptive-capacity.json" "$samples" "$resources" "$highest" "$failed"
|
||||
echo "local_acceptance_adaptive=PASS run_id=$run_id highest_stable_concurrency=$highest first_failed_concurrency=$failed report=$report_root/adaptive-capacity.json"
|
||||
}
|
||||
|
||||
full() {
|
||||
local manifest=$1 profile repetition
|
||||
quick
|
||||
@@ -610,7 +966,7 @@ full() {
|
||||
command=${1:-}
|
||||
shift || true
|
||||
case $command in
|
||||
quick)
|
||||
quick|adaptive|provider-burst)
|
||||
[[ $# -eq 0 ]] || { usage >&2; exit 64; }
|
||||
;;
|
||||
artifact-smoke|full)
|
||||
@@ -628,6 +984,12 @@ case $command in
|
||||
quick)
|
||||
quick
|
||||
;;
|
||||
adaptive)
|
||||
adaptive
|
||||
;;
|
||||
provider-burst)
|
||||
provider_burst
|
||||
;;
|
||||
artifact-smoke)
|
||||
artifact_smoke "$2"
|
||||
;;
|
||||
|
||||
Reference in New Issue
Block a user