feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -41,7 +41,8 @@ load_lock() {
|
||||
}
|
||||
# shellcheck source=/dev/null
|
||||
source "$lock_file"
|
||||
: "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
|
||||
: "${K3D_VERSION:?}" "${CRANE_VERSION:?}" "${K3S_IMAGE:?}" "${K3S_PAUSE_IMAGE:?}" "${K3S_BUSYBOX_IMAGE:?}"
|
||||
: "${CNPG_MANIFEST_URL:?}" "${CNPG_MANIFEST_SHA256:?}"
|
||||
: "${CNPG_CONTROLLER_IMAGE:?}" "${CNPG_POSTGRES_IMAGE:?}" "${K3S_COREDNS_IMAGE:?}"
|
||||
: "${K3S_METRICS_SERVER_IMAGE:?}" "${K3S_LOCAL_PATH_IMAGE:?}"
|
||||
}
|
||||
@@ -123,6 +124,9 @@ host_available_disk_bytes() {
|
||||
}
|
||||
|
||||
preflight() {
|
||||
local required_disk_gib=${1:-30} required_disk_bytes
|
||||
[[ $required_disk_gib =~ ^[0-9]+$ && $required_disk_gib -ge 10 ]]
|
||||
required_disk_bytes=$((required_disk_gib * 1024 * 1024 * 1024))
|
||||
load_lock
|
||||
require_command docker
|
||||
require_command kubectl
|
||||
@@ -165,12 +169,12 @@ preflight() {
|
||||
exit 1
|
||||
}
|
||||
available_disk_bytes=$(host_available_disk_bytes)
|
||||
[[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge 32212254720 ]] || {
|
||||
[[ $available_disk_bytes =~ ^[0-9]+$ && $available_disk_bytes -ge $required_disk_bytes ]] || {
|
||||
available_disk_gib=$(awk -v bytes="${available_disk_bytes:-0}" 'BEGIN {printf "%.1f", bytes/1024/1024/1024}')
|
||||
echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least 30 GiB" >&2
|
||||
echo "host free disk is ${available_disk_gib} GiB; local acceptance requires at least ${required_disk_gib} GiB" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes architecture=$architecture k3d=$K3D_VERSION"
|
||||
echo "local_acceptance_preflight=PASS docker_memory_bytes=$memory_bytes docker_cpus=$cpu_count host_available_disk_bytes=$available_disk_bytes required_disk_gib=$required_disk_gib architecture=$architecture k3d=$K3D_VERSION"
|
||||
}
|
||||
|
||||
ensure_private_material() {
|
||||
@@ -224,8 +228,8 @@ render_k3d_config() {
|
||||
host_memory_mib=$((host_memory_bytes / 1024 / 1024))
|
||||
site_system_cpu="$(((host_cpu - 4) * 1000 + 750))m"
|
||||
witness_system_cpu="$(((host_cpu - 2) * 1000 + 750))m"
|
||||
site_system_memory="$((host_memory_mib - 8192 + 1024))Mi"
|
||||
witness_system_memory="$((host_memory_mib - 4096 + 1024))Mi"
|
||||
site_system_memory="$((host_memory_mib - 8192 + 512))Mi"
|
||||
witness_system_memory="$((host_memory_mib - 4096 + 512))Mi"
|
||||
sed \
|
||||
-e "s|EASYAI_ACCEPTANCE_MEDIA_DIR|$media_root|g" \
|
||||
-e "s|EASYAI_ACCEPTANCE_K3S_IMAGE|${K3S_IMAGE%%@*}|g" \
|
||||
@@ -294,7 +298,8 @@ remove_cluster_network() {
|
||||
|
||||
pull_dependency_images() {
|
||||
local image
|
||||
for image in "$K3S_IMAGE" "$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
|
||||
for image in "$K3S_IMAGE" "$K3S_PAUSE_IMAGE" "$K3S_BUSYBOX_IMAGE" \
|
||||
"$CNPG_CONTROLLER_IMAGE" "$CNPG_POSTGRES_IMAGE" \
|
||||
"$K3S_COREDNS_IMAGE" "$K3S_METRICS_SERVER_IMAGE" "$K3S_LOCAL_PATH_IMAGE"; do
|
||||
pull_dependency_image "$image"
|
||||
done
|
||||
@@ -596,6 +601,10 @@ render_and_apply_application() {
|
||||
-f "$repository_root/deploy/kubernetes/production/service-account-rbac.yaml" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" apply -f "$manifest_root/local-config.yaml" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" apply -f "$rendered" >/dev/null
|
||||
for workload in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" patch deployment "$workload" \
|
||||
--type=merge -p='{"spec":{"template":{"spec":{"affinity":null}}}}' >/dev/null
|
||||
done
|
||||
|
||||
for workload in easyai-api-ningbo easyai-worker-ningbo; do
|
||||
kubectl --context "$context" -n "$namespace" set env deployment/"$workload" \
|
||||
@@ -791,10 +800,14 @@ up_cluster() {
|
||||
echo 'local K3s node allocatable capacity does not match 4/8, 4/8, 2/4 resource envelope' >&2
|
||||
exit 1
|
||||
}
|
||||
"$k3d" image import -c "$cluster_name" \
|
||||
local cluster_image
|
||||
for cluster_image in \
|
||||
"$api_image" "$web_image" "$netem_image" \
|
||||
"${K3S_PAUSE_IMAGE%%@*}" "${K3S_BUSYBOX_IMAGE%%@*}" \
|
||||
"${CNPG_CONTROLLER_IMAGE%%@*}" "${CNPG_POSTGRES_IMAGE%%@*}" \
|
||||
"${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"
|
||||
"${K3S_COREDNS_IMAGE%%@*}" "${K3S_METRICS_SERVER_IMAGE%%@*}" "${K3S_LOCAL_PATH_IMAGE%%@*}"; do
|
||||
"$k3d" image import -c "$cluster_name" "$cluster_image"
|
||||
done
|
||||
|
||||
local cnpg_manifest="$state_root/cnpg.yaml"
|
||||
curl -fsSL "$CNPG_MANIFEST_URL" -o "$cnpg_manifest"
|
||||
@@ -844,7 +857,7 @@ up_cluster() {
|
||||
|
||||
new_run() {
|
||||
load_lock
|
||||
preflight
|
||||
preflight 10
|
||||
[[ -z $(git -C "$repository_root" status --short) ]] || {
|
||||
echo 'local acceptance requires a clean source tree for a fresh Run ID' >&2
|
||||
exit 1
|
||||
|
||||
@@ -61,11 +61,12 @@ verify_local_node_capacity() {
|
||||
elif endswith("Gi") then rtrimstr("Gi") | tonumber * 1024
|
||||
else 0 end;
|
||||
([.items[] | select(.metadata.labels["easyai.io/workload"] == "true") |
|
||||
select(.metadata.labels["easyai.io/worker"] == "true") |
|
||||
select((.status.allocatable.cpu | cpu_m) == 3000 and
|
||||
(.status.allocatable.memory | memory_mi) == 6656)] | length) == 2 and
|
||||
(.status.allocatable.memory | memory_mi | floor) == 6656)] | length) == 2 and
|
||||
([.items[] | select(.metadata.labels["easyai.io/site"] == "los-angeles") |
|
||||
select((.status.allocatable.cpu | cpu_m) == 1000 and
|
||||
(.status.allocatable.memory | memory_mi) == 2560)] | length) == 1 and
|
||||
(.status.allocatable.memory | memory_mi | floor) == 2560)] | length) == 1 and
|
||||
([.items[] | select(any(.status.conditions[]; .type == "Ready" and .status == "True"))] | length) == 3 and
|
||||
([.items[] | select(any(.status.conditions[]; .type == "MemoryPressure" and .status == "True"))] | length) == 0
|
||||
' >/dev/null
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# This library is sourced by run-local-acceptance.sh and intentionally uses its
|
||||
# validated runtime globals and lifecycle state.
|
||||
# shellcheck disable=SC2034,SC2154
|
||||
|
||||
provider_burst_platform_ids=(
|
||||
f1000000-0000-4000-8000-000000000002
|
||||
f1000000-0000-4000-8000-000000000004
|
||||
f1000000-0000-4000-8000-000000000006
|
||||
)
|
||||
|
||||
provider_burst_model_ids=(
|
||||
f2000000-0000-4000-8000-000000000002
|
||||
f2000000-0000-4000-8000-000000000004
|
||||
f2000000-0000-4000-8000-000000000006
|
||||
)
|
||||
|
||||
restore_provider_burst_platforms() {
|
||||
database_query "
|
||||
UPDATE integration_platforms
|
||||
SET status = config->>'acceptanceBurstPreviousStatus',
|
||||
config = config - 'acceptanceBurstPreviousStatus',
|
||||
updated_at = statement_timestamp()
|
||||
WHERE config ? 'acceptanceBurstPreviousStatus';
|
||||
|
||||
UPDATE integration_platforms
|
||||
SET status = 'disabled',
|
||||
updated_at = statement_timestamp()
|
||||
WHERE COALESCE((config->>'acceptanceProviderBurst')::boolean, false);" >/dev/null
|
||||
}
|
||||
|
||||
setup_provider_burst_platforms() {
|
||||
[[ $run_id =~ ^[0-9a-f-]{36}$ ]]
|
||||
[[ $acceptance_video_model =~ ^[A-Za-z0-9._:-]+$ ]]
|
||||
restore_provider_burst_platforms
|
||||
database_query "
|
||||
WITH source AS (
|
||||
SELECT platform.*
|
||||
FROM integration_platforms platform
|
||||
JOIN platform_models model ON model.platform_id = platform.id
|
||||
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
||||
WHERE base_model.invocation_name = '$acceptance_video_model'
|
||||
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
||||
ORDER BY platform.priority, platform.created_at
|
||||
LIMIT 1
|
||||
), variants(platform_id, platform_key, platform_name, concurrency_limit) AS (
|
||||
VALUES
|
||||
('${provider_burst_platform_ids[0]}'::uuid, 'acceptance-provider-burst-2', 'Acceptance Provider Burst C2', 2),
|
||||
('${provider_burst_platform_ids[1]}'::uuid, 'acceptance-provider-burst-4', 'Acceptance Provider Burst C4', 4),
|
||||
('${provider_burst_platform_ids[2]}'::uuid, 'acceptance-provider-burst-6', 'Acceptance Provider Burst C6', 6)
|
||||
)
|
||||
INSERT INTO integration_platforms (
|
||||
id, provider, platform_key, name, base_url, auth_type, credentials, config,
|
||||
visibility_scope, tenant_id, tenant_key, default_pricing_mode,
|
||||
default_discount_factor, pricing_rule_set_id, retry_policy, rate_limit_policy,
|
||||
priority, dynamic_priority, status, disabled_reason, cooldown_until, internal_name
|
||||
)
|
||||
SELECT variants.platform_id, source.provider, variants.platform_key, variants.platform_name,
|
||||
source.base_url, source.auth_type, source.credentials,
|
||||
source.config || jsonb_build_object(
|
||||
'acceptanceProviderBurst', true,
|
||||
'acceptanceBurstRunId', '$run_id',
|
||||
'acceptanceConcurrencyLimit', variants.concurrency_limit
|
||||
),
|
||||
source.visibility_scope, source.tenant_id, source.tenant_key,
|
||||
source.default_pricing_mode, source.default_discount_factor,
|
||||
source.pricing_rule_set_id, source.retry_policy, '{\"rules\":[]}'::jsonb,
|
||||
100, NULL, 'enabled', NULL, NULL, variants.platform_name
|
||||
FROM source CROSS JOIN variants
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
provider = EXCLUDED.provider,
|
||||
platform_key = EXCLUDED.platform_key,
|
||||
name = EXCLUDED.name,
|
||||
base_url = EXCLUDED.base_url,
|
||||
auth_type = EXCLUDED.auth_type,
|
||||
credentials = EXCLUDED.credentials,
|
||||
config = EXCLUDED.config,
|
||||
visibility_scope = EXCLUDED.visibility_scope,
|
||||
tenant_id = EXCLUDED.tenant_id,
|
||||
tenant_key = EXCLUDED.tenant_key,
|
||||
default_pricing_mode = EXCLUDED.default_pricing_mode,
|
||||
default_discount_factor = EXCLUDED.default_discount_factor,
|
||||
pricing_rule_set_id = EXCLUDED.pricing_rule_set_id,
|
||||
retry_policy = EXCLUDED.retry_policy,
|
||||
rate_limit_policy = EXCLUDED.rate_limit_policy,
|
||||
priority = EXCLUDED.priority,
|
||||
dynamic_priority = EXCLUDED.dynamic_priority,
|
||||
status = 'enabled',
|
||||
disabled_reason = NULL,
|
||||
cooldown_until = NULL,
|
||||
internal_name = EXCLUDED.internal_name,
|
||||
deleted_at = NULL,
|
||||
updated_at = statement_timestamp();
|
||||
|
||||
WITH source AS (
|
||||
SELECT model.*
|
||||
FROM platform_models model
|
||||
JOIN integration_platforms platform ON platform.id = model.platform_id
|
||||
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
||||
WHERE base_model.invocation_name = '$acceptance_video_model'
|
||||
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
||||
ORDER BY platform.priority, platform.created_at
|
||||
LIMIT 1
|
||||
), variants(model_id, platform_id, concurrency_limit) AS (
|
||||
VALUES
|
||||
('${provider_burst_model_ids[0]}'::uuid, '${provider_burst_platform_ids[0]}'::uuid, 2),
|
||||
('${provider_burst_model_ids[1]}'::uuid, '${provider_burst_platform_ids[1]}'::uuid, 4),
|
||||
('${provider_burst_model_ids[2]}'::uuid, '${provider_burst_platform_ids[2]}'::uuid, 6)
|
||||
)
|
||||
INSERT INTO platform_models (
|
||||
id, platform_id, base_model_id, model_name, model_alias, model_type,
|
||||
display_name, capability_override, capabilities, pricing_mode,
|
||||
discount_factor, pricing_rule_set_id, billing_config_override, billing_config,
|
||||
permission_config, retry_policy, rate_limit_policy, rate_limit_policy_mode,
|
||||
runtime_policy_set_id, runtime_policy_override, provider_model_name,
|
||||
cooldown_until, enabled
|
||||
)
|
||||
SELECT variants.model_id, variants.platform_id, source.base_model_id,
|
||||
source.model_name, source.model_alias, source.model_type, source.display_name,
|
||||
source.capability_override, source.capabilities, source.pricing_mode,
|
||||
source.discount_factor, source.pricing_rule_set_id,
|
||||
source.billing_config_override, source.billing_config,
|
||||
source.permission_config, source.retry_policy,
|
||||
jsonb_build_object('rules', jsonb_build_array(
|
||||
jsonb_build_object(
|
||||
'metric', 'concurrent', 'limit', variants.concurrency_limit,
|
||||
'leaseTtlSeconds', 120
|
||||
),
|
||||
jsonb_build_object('metric', 'rpm', 'limit', 600, 'windowSeconds', 60)
|
||||
)),
|
||||
'override', NULL, '{}'::jsonb, source.provider_model_name, NULL, true
|
||||
FROM source CROSS JOIN variants
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
platform_id = EXCLUDED.platform_id,
|
||||
base_model_id = EXCLUDED.base_model_id,
|
||||
model_name = EXCLUDED.model_name,
|
||||
model_alias = EXCLUDED.model_alias,
|
||||
model_type = EXCLUDED.model_type,
|
||||
display_name = EXCLUDED.display_name,
|
||||
capability_override = EXCLUDED.capability_override,
|
||||
capabilities = EXCLUDED.capabilities,
|
||||
pricing_mode = EXCLUDED.pricing_mode,
|
||||
discount_factor = EXCLUDED.discount_factor,
|
||||
pricing_rule_set_id = EXCLUDED.pricing_rule_set_id,
|
||||
billing_config_override = EXCLUDED.billing_config_override,
|
||||
billing_config = EXCLUDED.billing_config,
|
||||
permission_config = EXCLUDED.permission_config,
|
||||
retry_policy = EXCLUDED.retry_policy,
|
||||
rate_limit_policy = EXCLUDED.rate_limit_policy,
|
||||
rate_limit_policy_mode = 'override',
|
||||
runtime_policy_set_id = NULL,
|
||||
runtime_policy_override = '{}'::jsonb,
|
||||
provider_model_name = EXCLUDED.provider_model_name,
|
||||
cooldown_until = NULL,
|
||||
enabled = true,
|
||||
updated_at = statement_timestamp();
|
||||
|
||||
WITH selected_platforms AS (
|
||||
SELECT platform.id
|
||||
FROM integration_platforms platform
|
||||
JOIN platform_models model ON model.platform_id = platform.id
|
||||
JOIN base_model_catalog base_model ON base_model.id = model.base_model_id
|
||||
WHERE base_model.invocation_name = '$acceptance_video_model'
|
||||
AND COALESCE((platform.config->>'acceptanceSnapshot')::boolean, false)
|
||||
AND NOT COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
)
|
||||
UPDATE integration_platforms platform
|
||||
SET config = jsonb_set(
|
||||
platform.config,
|
||||
'{acceptanceBurstPreviousStatus}',
|
||||
to_jsonb(platform.status),
|
||||
true
|
||||
),
|
||||
status = 'disabled',
|
||||
updated_at = statement_timestamp()
|
||||
FROM selected_platforms
|
||||
WHERE platform.id = selected_platforms.id;
|
||||
|
||||
WITH acceptance_group AS (
|
||||
SELECT api_key.user_group_id AS id
|
||||
FROM gateway_acceptance_runs run
|
||||
JOIN gateway_api_keys api_key ON api_key.id::text = run.api_key_id
|
||||
WHERE run.id = '$run_id'::uuid
|
||||
), resources(resource_type, resource_id) AS (
|
||||
SELECT 'platform'::text, platform.id
|
||||
FROM integration_platforms platform
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
||||
UNION ALL
|
||||
SELECT 'platform_model', model.id
|
||||
FROM platform_models model
|
||||
JOIN integration_platforms platform ON platform.id = model.platform_id
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
||||
UNION
|
||||
SELECT 'base_model', model.base_model_id
|
||||
FROM platform_models model
|
||||
JOIN integration_platforms platform ON platform.id = model.platform_id
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
||||
)
|
||||
INSERT INTO gateway_access_rules (
|
||||
subject_type, subject_id, resource_type, resource_id, effect,
|
||||
priority, min_permission_level, conditions, metadata, status
|
||||
)
|
||||
SELECT 'user_group', acceptance_group.id, resources.resource_type,
|
||||
resources.resource_id, 'allow', 1, 0, '{}'::jsonb,
|
||||
jsonb_build_object('purpose', 'provider_burst_acceptance', 'runId', '$run_id'),
|
||||
'active'
|
||||
FROM acceptance_group CROSS JOIN resources
|
||||
WHERE acceptance_group.id IS NOT NULL
|
||||
ON CONFLICT (subject_type, subject_id, resource_type, resource_id, effect)
|
||||
DO UPDATE SET status = 'active', metadata = EXCLUDED.metadata,
|
||||
updated_at = statement_timestamp();" >/dev/null
|
||||
|
||||
local configured
|
||||
configured=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'platforms', count(*),
|
||||
'limits', jsonb_agg((platform.config->>'acceptanceConcurrencyLimit')::int ORDER BY (platform.config->>'acceptanceConcurrencyLimit')::int),
|
||||
'enabled', count(*) FILTER (WHERE platform.status='enabled' AND model.enabled)
|
||||
)
|
||||
FROM integration_platforms platform
|
||||
JOIN platform_models model ON model.platform_id = platform.id
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId' = '$run_id';")
|
||||
jq -e '.platforms == 3 and .enabled == 3 and .limits == [2,4,6]' <<<"$configured" >/dev/null
|
||||
}
|
||||
|
||||
sample_provider_burst() {
|
||||
local output=$1 stop_file=$2
|
||||
: >"$output"
|
||||
while [[ ! -e $stop_file ]]; do
|
||||
database_query "
|
||||
WITH configured AS (
|
||||
SELECT platform.id platform_id, platform.platform_key, model.id platform_model_id,
|
||||
(platform.config->>'acceptanceConcurrencyLimit')::int concurrency_limit
|
||||
FROM integration_platforms platform
|
||||
JOIN platform_models model ON model.platform_id = platform.id
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId' = '$run_id'
|
||||
), platform_state AS (
|
||||
SELECT configured.*,
|
||||
(SELECT count(*) FROM gateway_concurrency_leases lease
|
||||
WHERE lease.scope_type='platform_model'
|
||||
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
||||
AND lease.released_at IS NULL
|
||||
AND lease.expires_at > statement_timestamp()) active_leases,
|
||||
(SELECT count(*) FROM gateway_task_attempts attempt
|
||||
WHERE attempt.platform_model_id=configured.platform_model_id
|
||||
AND attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)) attempts
|
||||
FROM configured
|
||||
)
|
||||
SELECT jsonb_build_object(
|
||||
'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'),
|
||||
'queuedTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='queued'),
|
||||
'runningTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='running'),
|
||||
'succeededTasks', (SELECT count(*) FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND status='succeeded'),
|
||||
'waitingAdmissions', (SELECT count(*) FROM gateway_task_admissions admission JOIN gateway_tasks task ON task.id=admission.task_id WHERE task.acceptance_run_id='$run_id'::uuid AND admission.status='waiting'),
|
||||
'platforms', COALESCE((SELECT jsonb_agg(jsonb_build_object(
|
||||
'platformId', platform_id,
|
||||
'platformModelId', platform_model_id,
|
||||
'platformKey', platform_key,
|
||||
'limit', concurrency_limit,
|
||||
'activeLeases', active_leases,
|
||||
'attempts', attempts
|
||||
) ORDER BY concurrency_limit) FROM platform_state), '[]'::jsonb)
|
||||
);" >>"$output" || true
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
provider_burst_resource_summary() {
|
||||
local resources=$1
|
||||
jq -Rn '
|
||||
def number_without($suffix): rtrimstr($suffix) | tonumber;
|
||||
def memory_mib:
|
||||
if endswith("Gi") then number_without("Gi") * 1024
|
||||
elif endswith("Mi") then number_without("Mi")
|
||||
elif endswith("Ki") then number_without("Ki") / 1024
|
||||
else tonumber end;
|
||||
[inputs | split(",") | select(.[0] != "timestamp") | {
|
||||
scope:.[1],name:.[2],cpu:.[3],memory:.[4]
|
||||
}]
|
||||
| group_by([.scope,.name])
|
||||
| map(if .[0].scope == "node" then {
|
||||
scope:"node",name:.[0].name,
|
||||
maxCpuPercent:(map(.cpu | number_without("%")) | max),
|
||||
maxMemoryPercent:(map(.memory | number_without("%")) | max)
|
||||
} else {
|
||||
scope:"pod",name:.[0].name,
|
||||
maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max),
|
||||
maxMemoryMiB:(map(.memory | memory_mib) | max)
|
||||
} end)' <"$resources"
|
||||
}
|
||||
|
||||
build_provider_burst_report() {
|
||||
local output=$1 load_report=$2 samples=$3 worker_samples=$4 resources=$5 requests=$6
|
||||
local platforms queue workers distribution worker_peaks limits concurrency_peaks
|
||||
local leaks duplicates callbacks resource_summary passed=true
|
||||
platforms=$(database_query "
|
||||
WITH configured AS (
|
||||
SELECT platform.id platform_id, platform.platform_key, model.id platform_model_id,
|
||||
(platform.config->>'acceptanceConcurrencyLimit')::int concurrency_limit
|
||||
FROM integration_platforms platform
|
||||
JOIN platform_models model ON model.platform_id=platform.id
|
||||
WHERE COALESCE((platform.config->>'acceptanceProviderBurst')::boolean, false)
|
||||
AND platform.config->>'acceptanceBurstRunId'='$run_id'
|
||||
), events AS (
|
||||
SELECT configured.platform_model_id, lease.acquired_at event_at, lease.lease_value delta
|
||||
FROM configured
|
||||
JOIN gateway_concurrency_leases lease
|
||||
ON lease.scope_type='platform_model'
|
||||
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
||||
UNION ALL
|
||||
SELECT configured.platform_model_id,
|
||||
LEAST(COALESCE(lease.released_at, lease.expires_at), lease.expires_at),
|
||||
-lease.lease_value
|
||||
FROM configured
|
||||
JOIN gateway_concurrency_leases lease
|
||||
ON lease.scope_type='platform_model'
|
||||
AND lease.scope_key='acceptance:$run_id:'||configured.platform_model_id::text
|
||||
), points AS (
|
||||
SELECT platform_model_id,
|
||||
SUM(delta) OVER (PARTITION BY platform_model_id ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
||||
FROM events
|
||||
), peaks AS (
|
||||
SELECT platform_model_id, COALESCE(max(active),0) peak FROM points GROUP BY platform_model_id
|
||||
), attempts AS (
|
||||
SELECT attempt.platform_model_id, count(*) attempts
|
||||
FROM gateway_task_attempts attempt
|
||||
WHERE attempt.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
||||
GROUP BY attempt.platform_model_id
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'platformId', configured.platform_id,
|
||||
'platformModelId', configured.platform_model_id,
|
||||
'platformKey', configured.platform_key,
|
||||
'limit', configured.concurrency_limit,
|
||||
'peak', COALESCE(peaks.peak,0),
|
||||
'attempts', COALESCE(attempts.attempts,0)
|
||||
) ORDER BY configured.concurrency_limit), '[]'::jsonb)
|
||||
FROM configured
|
||||
LEFT JOIN peaks ON peaks.platform_model_id=configured.platform_model_id
|
||||
LEFT JOIN attempts ON attempts.platform_model_id=configured.platform_model_id;")
|
||||
queue=$(jq -s '{
|
||||
samples:length,
|
||||
maxQueuedTasks:(map(.queuedTasks)|max),
|
||||
maxRunningTasks:(map(.runningTasks)|max),
|
||||
maxWaitingAdmissions:(map(.waitingAdmissions)|max),
|
||||
queueObserved:any(.queuedTasks > 0),
|
||||
admissionWaitObserved:any(.waitingAdmissions > 0),
|
||||
drainedAtEnd:(.[-1].queuedTasks == 0 and .[-1].runningTasks == 0 and .[-1].waitingAdmissions == 0)
|
||||
}' "$samples")
|
||||
workers=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instanceId',instance_id,'podName',pod_name,'site',site,
|
||||
'allocatedCapacity',allocated_capacity,'safeCapacity',safe_capacity,
|
||||
'heavyCapacity',heavy_capacity,'pressureState',pressure_state
|
||||
) ORDER BY site,instance_id),'[]'::jsonb)
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';")
|
||||
distribution=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb)
|
||||
FROM (
|
||||
SELECT COALESCE(
|
||||
regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''),
|
||||
'unclaimed'
|
||||
) worker_id,
|
||||
count(*) tasks
|
||||
FROM gateway_tasks task
|
||||
LEFT JOIN river_job job ON job.id=task.river_job_id
|
||||
WHERE task.acceptance_run_id='$run_id'::uuid
|
||||
GROUP BY worker_id
|
||||
) grouped;")
|
||||
worker_peaks=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({
|
||||
instanceId:.[0].instanceId,
|
||||
peakActiveTasks:(map(.activeTasks)|max),
|
||||
peakPreparingTasks:(map(.preparingTasks)|max),
|
||||
peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max),
|
||||
peakFinalizingTasks:(map(.finalizingTasks)|max),
|
||||
minSafeCapacity:(map(.safeCapacity)|min),
|
||||
maxSafeCapacity:(map(.safeCapacity)|max),
|
||||
pressureStates:(map(.pressureState)|unique)
|
||||
})' "$worker_samples")
|
||||
limits=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'scopeType',scope_type,'scopeKey',scope_key,'metric',metric,
|
||||
'limit',limit_value,'used',used_value,'reserved',reserved_value,
|
||||
'windowStart',window_start,'resetAt',reset_at
|
||||
) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb)
|
||||
FROM gateway_rate_limit_counters
|
||||
WHERE scope_key LIKE 'acceptance:$run_id:%';")
|
||||
concurrency_peaks=$(database_query "
|
||||
WITH events AS (
|
||||
SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta
|
||||
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
UNION ALL
|
||||
SELECT scope_type,scope_key,limit_value,
|
||||
LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value
|
||||
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
), points AS (
|
||||
SELECT scope_type,scope_key,limit_value,
|
||||
SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
||||
FROM events
|
||||
), peaks AS (
|
||||
SELECT scope_type,scope_key,max(limit_value) limit_value,COALESCE(max(active),0) peak
|
||||
FROM points GROUP BY scope_type,scope_key
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak
|
||||
) ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;")
|
||||
leaks=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>statement_timestamp()),
|
||||
'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'),
|
||||
'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled'))
|
||||
);")
|
||||
duplicates=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate),
|
||||
'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate)
|
||||
);")
|
||||
callbacks=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'deliveries',count(*) FILTER (WHERE callback.status='delivered'),
|
||||
'pending',count(*) FILTER (WHERE callback.status<>'delivered'),
|
||||
'duplicates',(SELECT count(*) FROM (
|
||||
SELECT callback_inner.task_id,callback_inner.seq,callback_inner.callback_url
|
||||
FROM gateway_task_callback_outbox callback_inner
|
||||
WHERE callback_inner.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
||||
GROUP BY callback_inner.task_id,callback_inner.seq,callback_inner.callback_url
|
||||
HAVING count(*)>1
|
||||
) duplicate)
|
||||
)
|
||||
FROM gateway_task_callback_outbox callback
|
||||
WHERE callback.task_id IN (SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid);")
|
||||
resource_summary=$(provider_burst_resource_summary "$resources")
|
||||
if ! jq -e 'length==3 and all(.[]; .peak == .limit and .attempts > 0)' <<<"$platforms" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.queueObserved and .admissionWaitObserved and .drainedAtEnd and .maxQueuedTasks > 0 and .maxWaitingAdmissions > 0' <<<"$queue" >/dev/null; then passed=false; fi
|
||||
if ! jq -e --argjson requests "$requests" 'length==3 and all(.[]; .tasks>0 and .workerInstanceId!="unclaimed") and ([.[].tasks]|add)==$requests' <<<"$distribution" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'length==3 and all(.[]; .peakActiveTasks>0 and (.pressureStates|index("critical")|not))' <<<"$worker_peaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'length>0 and all(.[]; (.used+.reserved)<=.limit)' <<<"$limits" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'all(.[]; .peak<=.limit)' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.deliveries>0 and .pending==0 and .duplicates==0' <<<"$callbacks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e --argjson requests "$requests" '.passed and ([.phases[].completed]|add)==$requests and ([.phases[].failed]|add)==0' "$load_report" >/dev/null; then passed=false; fi
|
||||
jq -n \
|
||||
--arg runId "$run_id" --arg model "$acceptance_video_model" \
|
||||
--argjson passed "$passed" --argjson requests "$requests" \
|
||||
--argjson phases "$(jq '.phases' "$load_report")" \
|
||||
--argjson platforms "$platforms" --argjson queue "$queue" \
|
||||
--argjson workers "$workers" --argjson distribution "$distribution" \
|
||||
--argjson workerPeaks "$worker_peaks" --argjson rateLimits "$limits" \
|
||||
--argjson concurrencyPeaks "$concurrency_peaks" \
|
||||
--argjson resources "$resource_summary" --argjson leaks "$leaks" \
|
||||
--argjson duplicates "$duplicates" --argjson callbacks "$callbacks" \
|
||||
'{
|
||||
schemaVersion:"acceptance-provider-burst-report/v1",
|
||||
runId:$runId,profile:"provider-burst",model:$model,
|
||||
startedAt:(now|todateiso8601),finishedAt:(now|todateiso8601),
|
||||
passed:$passed,secretSafe:true,requests:$requests,phases:$phases,
|
||||
providerRouting:{platforms:$platforms,totalConfiguredConcurrency:([$platforms[].limit]|add)},
|
||||
queue:$queue,
|
||||
cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,
|
||||
rateLimits:$rateLimits,concurrencyPeaks:$concurrencyPeaks,
|
||||
resourcePeaks:$resources,leaks:$leaks,duplicates:$duplicates,callbacks:$callbacks}
|
||||
}' >"$output"
|
||||
chmod 0600 "$output"
|
||||
jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null
|
||||
}
|
||||
|
||||
provider_burst() {
|
||||
local requests=${AI_GATEWAY_LOCAL_PROVIDER_BURST_REQUESTS:-48}
|
||||
local load_report="$report_root/provider-burst-load.json"
|
||||
local samples="$report_root/provider-burst-samples.ndjson"
|
||||
local worker_samples="$report_root/provider-burst-worker-samples.ndjson"
|
||||
local resources="$report_root/provider-burst-resources.csv"
|
||||
local report="$report_root/provider-burst.json"
|
||||
worker_sample_stop="$report_root/provider-burst-samples.stop"
|
||||
[[ $requests =~ ^[0-9]+$ ]] && ((requests >= 24 && requests <= 256))
|
||||
[[ ! -e $load_report && ! -e $samples && ! -e $worker_samples && ! -e $resources && ! -e $report && ! -e $worker_sample_stop ]] ||
|
||||
fail_gate acceptance_report_exists "provider burst evidence already exists"
|
||||
current_phase=provider_burst_platforms
|
||||
setup_provider_burst_platforms
|
||||
provider_burst_configured=true
|
||||
current_phase=provider_burst_workers
|
||||
configure_adaptive_workers
|
||||
sample_provider_burst "$samples" "$worker_sample_stop" &
|
||||
active_provider_sampler_pid=$!
|
||||
sample_worker_runtime "$worker_samples" "$worker_sample_stop" &
|
||||
active_worker_sampler_pid=$!
|
||||
sample_resources "$resources" "$worker_sample_stop" &
|
||||
active_resource_sampler_pid=$!
|
||||
current_phase=provider_burst_load
|
||||
run_load video-throughput "$load_report" -requests "$requests"
|
||||
sleep 2
|
||||
touch "$worker_sample_stop"
|
||||
wait "$active_provider_sampler_pid"
|
||||
wait "$active_worker_sampler_pid"
|
||||
wait "$active_resource_sampler_pid"
|
||||
active_provider_sampler_pid=
|
||||
active_worker_sampler_pid=
|
||||
active_resource_sampler_pid=
|
||||
worker_sample_stop=
|
||||
current_phase=provider_burst_hard_gates
|
||||
verify_hard_gates
|
||||
current_phase=provider_burst_report
|
||||
build_provider_burst_report "$report" "$load_report" "$samples" "$worker_samples" "$resources" "$requests"
|
||||
restore_provider_burst_platforms
|
||||
provider_burst_configured=false
|
||||
echo "local_acceptance_provider_burst=PASS run_id=$run_id requests=$requests report=$report"
|
||||
}
|
||||
@@ -154,7 +154,7 @@ async function loadReports(directory) {
|
||||
const reports = [];
|
||||
for (const name of entries) {
|
||||
const report = await regularJSON(resolve(directory, name));
|
||||
if (report.schemaVersion === 'acceptance-load-report/v1') {
|
||||
if (report.schemaVersion === 'acceptance-load-report/v1' || report.schemaVersion === 'acceptance-load-report/v2') {
|
||||
if (report.secretSafe !== true) throw new Error(`load report ${name} is not secret-safe`);
|
||||
reports.push({
|
||||
file: name,
|
||||
@@ -163,6 +163,8 @@ async function loadReports(directory) {
|
||||
phases: report.phases,
|
||||
failure: report.failure,
|
||||
failureOperation: report.failureOperation,
|
||||
capacity: report.capacity,
|
||||
cluster: report.cluster,
|
||||
startedAt: report.startedAt,
|
||||
finishedAt: report.finishedAt
|
||||
});
|
||||
|
||||
@@ -13,20 +13,32 @@ snapshot="$state_root/snapshot.json"
|
||||
load_binary="$state_root/easyai-ai-gateway-acceptance-load"
|
||||
identity_file="$state_root/control-plane-identity.json"
|
||||
active_load_pid=
|
||||
active_worker_sampler_pid=
|
||||
active_resource_sampler_pid=
|
||||
active_provider_sampler_pid=
|
||||
worker_sample_stop=
|
||||
netem_active=false
|
||||
provider_burst_configured=false
|
||||
current_phase=initialization
|
||||
stable_profile=
|
||||
failure_gate_id=local_execution_incomplete
|
||||
# shellcheck source=scripts/acceptance/local-control-plane.sh
|
||||
source "$script_dir/local-control-plane.sh"
|
||||
# shellcheck source=scripts/acceptance/provider-burst.sh
|
||||
source "$script_dir/provider-burst.sh"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
scripts/acceptance/run-local-acceptance.sh quick
|
||||
scripts/acceptance/run-local-acceptance.sh adaptive
|
||||
scripts/acceptance/run-local-acceptance.sh provider-burst
|
||||
scripts/acceptance/run-local-acceptance.sh full --release-manifest dist/releases/<SHA>.json
|
||||
scripts/acceptance/run-local-acceptance.sh artifact-smoke --release-manifest dist/releases/<SHA>.json
|
||||
|
||||
`adaptive` runs a three-Worker emulator-only concurrency ladder without changing
|
||||
provider limits. `provider-burst` binds one video model to three emulator-only
|
||||
platforms with concurrency limits 2/4/6 and validates burst queueing and routing.
|
||||
`full` executes P24/P28/P32 three times, the fault matrix, autoscaling/drain,
|
||||
80% soak, 120% overload, and exact linux/amd64 artifact smoke. The load process
|
||||
runs outside K3s and splits requests 50/50 across both TLS entrances.
|
||||
@@ -51,9 +63,24 @@ cleanup() {
|
||||
kill "$active_load_pid" >/dev/null 2>&1 || true
|
||||
wait "$active_load_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_worker_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_worker_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_resource_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_resource_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ -n $active_provider_sampler_pid ]]; then
|
||||
[[ -z $worker_sample_stop ]] || touch "$worker_sample_stop"
|
||||
wait "$active_provider_sampler_pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $netem_active == true ]]; then
|
||||
"$script_dir/network-fault.sh" reset >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $provider_burst_configured == true && -n ${runtime:-} ]]; then
|
||||
restore_provider_burst_platforms >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $status -ne 0 && -n ${runtime:-} && -n ${report_root:-} && -f ${runtime:-} && -f $snapshot ]]; then
|
||||
restore_profile_best_effort "${stable_profile:-P24}"
|
||||
if [[ -f ${failure_gate_file:-} && ! -L ${failure_gate_file:-} ]]; then
|
||||
@@ -206,10 +233,281 @@ run_load() {
|
||||
return 1
|
||||
fi
|
||||
((load_status == 0)) || return "$load_status"
|
||||
jq -e '.schemaVersion == "acceptance-load-report/v1" and .secretSafe == true and .passed == true' \
|
||||
jq -e '.schemaVersion == "acceptance-load-report/v2" and .secretSafe == true and .passed == true' \
|
||||
"$report_path" >/dev/null
|
||||
}
|
||||
|
||||
sample_worker_runtime() {
|
||||
local output=$1 stop_file=$2
|
||||
: >"$output"
|
||||
while [[ ! -e $stop_file ]]; do
|
||||
database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'sampledAt', to_char(clock_timestamp() AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS.MS\"Z\"'),
|
||||
'workers', COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instanceId', instance_id,
|
||||
'allocatedCapacity', allocated_capacity,
|
||||
'safeCapacity', safe_capacity,
|
||||
'heavyCapacity', heavy_capacity,
|
||||
'activeTasks', active_tasks,
|
||||
'preparingTasks', preparing_tasks,
|
||||
'waitingUpstreamTasks', waiting_upstream_tasks,
|
||||
'finalizingTasks', finalizing_tasks,
|
||||
'pressureState', pressure_state,
|
||||
'pressureReason', pressure_reason
|
||||
) ORDER BY instance_id), '[]'::jsonb)
|
||||
)
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';" >>"$output" || true
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_adaptive_workloads_settled() {
|
||||
local deadline=$((SECONDS + 180)) stable_samples=0 pod_state
|
||||
local api_total api_ready api_terminating worker_total worker_ready worker_terminating
|
||||
while ((SECONDS < deadline)); do
|
||||
pod_state=$(kubectl --context "$context" -n "$namespace" get pods \
|
||||
-l 'app.kubernetes.io/name in (easyai-api,easyai-worker)' -o json) || {
|
||||
stable_samples=0
|
||||
sleep 2
|
||||
continue
|
||||
}
|
||||
read -r api_total api_ready api_terminating worker_total worker_ready worker_terminating < <(
|
||||
jq -r '
|
||||
def total($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp == null)] | length;
|
||||
def ready($name): [.items[] | select(
|
||||
.metadata.labels["app.kubernetes.io/name"] == $name and
|
||||
.metadata.deletionTimestamp == null and
|
||||
.status.phase == "Running" and
|
||||
((.status.containerStatuses // []) | length) > 0 and
|
||||
all(.status.containerStatuses[]; .ready == true)
|
||||
)] | length;
|
||||
def terminating($name): [.items[] | select(.metadata.labels["app.kubernetes.io/name"] == $name and .metadata.deletionTimestamp != null)] | length;
|
||||
[total("easyai-api"), ready("easyai-api"), terminating("easyai-api"), total("easyai-worker"), ready("easyai-worker"), terminating("easyai-worker")] | @tsv
|
||||
' <<<"$pod_state"
|
||||
)
|
||||
if [[ $api_total == 2 && $api_ready == 2 && $api_terminating == 0 &&
|
||||
$worker_total == 3 && $worker_ready == 3 && $worker_terminating == 0 ]]; then
|
||||
stable_samples=$((stable_samples + 1))
|
||||
if ((stable_samples >= 3)); then
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
stable_samples=0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
fail_gate adaptive_workload_settle_timeout \
|
||||
"adaptive workloads did not settle at two Ready APIs and three Ready Workers without terminating Pods"
|
||||
}
|
||||
|
||||
configure_adaptive_workers() {
|
||||
local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
|
||||
local global_limit=$((hard_limit * 3))
|
||||
[[ $hard_limit =~ ^[0-9]+$ ]] && ((hard_limit >= 4 && hard_limit <= 256))
|
||||
kubectl --context "$context" -n "$namespace" patch configmap easyai-ai-gateway-config \
|
||||
--type=merge -p "$(jq -cn --arg hard "$hard_limit" --arg global "$global_limit" '{data:{
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
||||
AI_GATEWAY_WORKER_LOAD_MODE:"adaptive",
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$hard,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$global,
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$global
|
||||
}}')" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout restart \
|
||||
deployment/easyai-capacity-controller >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status \
|
||||
deployment/easyai-capacity-controller --timeout=10m
|
||||
kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-ningbo --replicas=2 >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" scale deployment/easyai-worker-hongkong --replicas=1 >/dev/null
|
||||
for deployment in easyai-worker-ningbo easyai-worker-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" set env deployment/"$deployment" \
|
||||
AI_GATEWAY_WORKER_LOAD_MODE=adaptive \
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT="$hard_limit" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
for deployment in easyai-api-ningbo easyai-api-hongkong; do
|
||||
kubectl --context "$context" -n "$namespace" rollout restart deployment/"$deployment" >/dev/null
|
||||
kubectl --context "$context" -n "$namespace" rollout status deployment/"$deployment" --timeout=10m
|
||||
done
|
||||
wait_for_adaptive_workloads_settled
|
||||
local ready
|
||||
ready=$(kubectl --context "$context" -n "$namespace" get pods -l app.kubernetes.io/name=easyai-worker -o json |
|
||||
jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
|
||||
[[ $ready == 3 ]] || fail_gate adaptive_worker_count "adaptive validation requires exactly three Ready Workers"
|
||||
}
|
||||
|
||||
run_adaptive_level() {
|
||||
local concurrency=$1 passed=true etcd_healthy=true
|
||||
local gemini_report="$report_root/adaptive-load-${concurrency}-gemini.json"
|
||||
local video_report="$report_root/adaptive-load-${concurrency}-video.json"
|
||||
verify_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
||||
fail_gate local_control_plane_restarted "local K3s server identity changed before adaptive level $concurrency"
|
||||
record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" ||
|
||||
fail_gate local_control_plane_identity "failed to record adaptive level $concurrency control-plane baseline"
|
||||
if ! run_load gemini-multi-image "$gemini_report" -requests "$concurrency"; then
|
||||
passed=false
|
||||
fi
|
||||
if [[ $passed == true ]] && ! run_load video-throughput "$video_report" -requests "$concurrency"; then
|
||||
passed=false
|
||||
fi
|
||||
local critical
|
||||
critical=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND pressure_state='critical' AND heartbeat_at > now() - interval '30 seconds';")
|
||||
if ((critical > 0)); then
|
||||
passed=false
|
||||
fi
|
||||
if ! verify_local_etcd_runtime_logs "$cluster_name" "$identity_file"; then
|
||||
passed=false
|
||||
etcd_healthy=false
|
||||
record_local_control_plane_identity "$context" "$cluster_name" "$identity_file" || return 1
|
||||
fi
|
||||
jq -n --argjson concurrency "$concurrency" --argjson passed "$passed" \
|
||||
--argjson etcdHealthy "$etcd_healthy" \
|
||||
--arg gemini "$(basename "$gemini_report")" --arg video "$(basename "$video_report")" \
|
||||
'{concurrency:$concurrency,passed:$passed,etcdHealthy:$etcdHealthy,reports:[$gemini,$video]}' \
|
||||
>"$report_root/adaptive-level-${concurrency}.json"
|
||||
chmod 0600 "$report_root/adaptive-level-${concurrency}.json"
|
||||
[[ $passed == true ]]
|
||||
}
|
||||
|
||||
build_adaptive_report() {
|
||||
local output=$1 samples=$2 resources=$3 highest=$4 failed=${5:-0}
|
||||
local levels workers distribution routing limits concurrency_peaks leaks duplicates sample_summary resource_summary phases
|
||||
levels=$(jq -s 'sort_by(.concurrency)' "$report_root"/adaptive-level-*.json)
|
||||
workers=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instanceId',instance_id,'podName',pod_name,'site',site,'status',status,
|
||||
'allocatedCapacity',allocated_capacity,'hardCapacityLimit',hard_capacity_limit,
|
||||
'safeCapacity',safe_capacity,'heavyCapacity',heavy_capacity,
|
||||
'activeTasks',active_tasks,'preparingTasks',preparing_tasks,
|
||||
'waitingUpstreamTasks',waiting_upstream_tasks,'finalizingTasks',finalizing_tasks,
|
||||
'pressureState',pressure_state,'pressureReason',pressure_reason,
|
||||
'loadSampledAt',load_sampled_at
|
||||
) ORDER BY site,instance_id),'[]'::jsonb)
|
||||
FROM gateway_worker_instances
|
||||
WHERE heartbeat_at > now() - interval '30 seconds';")
|
||||
distribution=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object('workerInstanceId',worker_id,'tasks',tasks) ORDER BY worker_id),'[]'::jsonb)
|
||||
FROM (
|
||||
SELECT COALESCE(
|
||||
regexp_replace(job.attempted_by[cardinality(job.attempted_by)], '-exec-[^-]+-[^-]+$', ''),
|
||||
'unclaimed'
|
||||
) worker_id,
|
||||
count(*) tasks
|
||||
FROM gateway_tasks task
|
||||
LEFT JOIN river_job job ON job.id = task.river_job_id
|
||||
WHERE task.acceptance_run_id='$run_id'::uuid
|
||||
GROUP BY worker_id
|
||||
) grouped;")
|
||||
routing=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'platformId',platform_id,'platformModelId',platform_model_id,'attempts',attempts
|
||||
) ORDER BY platform_id,platform_model_id),'[]'::jsonb)
|
||||
FROM (SELECT platform_id::text,platform_model_id::text,count(*) attempts
|
||||
FROM gateway_task_attempts WHERE task_id IN
|
||||
(SELECT id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid)
|
||||
GROUP BY platform_id,platform_model_id) grouped;")
|
||||
limits=$(database_query "
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'scopeType',scope_type,'scopeKey',scope_key,'metric',metric,
|
||||
'limit',limit_value,'used',used_value,'reserved',reserved_value,
|
||||
'windowStart',window_start,'resetAt',reset_at
|
||||
) ORDER BY scope_type,scope_key,metric,window_start),'[]'::jsonb)
|
||||
FROM gateway_rate_limit_counters
|
||||
WHERE scope_key LIKE 'acceptance:$run_id:%';")
|
||||
concurrency_peaks=$(database_query "
|
||||
WITH events AS (
|
||||
SELECT scope_type,scope_key,limit_value,acquired_at event_at,lease_value delta FROM gateway_concurrency_leases
|
||||
WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
UNION ALL
|
||||
SELECT scope_type,scope_key,limit_value,LEAST(COALESCE(released_at,expires_at),expires_at),-lease_value
|
||||
FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%'
|
||||
), points AS (
|
||||
SELECT scope_type,scope_key,limit_value,
|
||||
SUM(delta) OVER (PARTITION BY scope_type,scope_key ORDER BY event_at,delta ROWS UNBOUNDED PRECEDING) active
|
||||
FROM events
|
||||
), peaks AS (
|
||||
SELECT scope_type,scope_key,MAX(limit_value) limit_value,COALESCE(MAX(active),0) peak
|
||||
FROM points GROUP BY scope_type,scope_key
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object('scopeType',scope_type,'scopeKey',scope_key,'limit',limit_value,'peak',peak)
|
||||
ORDER BY scope_type,scope_key),'[]'::jsonb) FROM peaks;")
|
||||
leaks=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'activeConcurrencyLeases',(SELECT count(*) FROM gateway_concurrency_leases WHERE scope_key LIKE 'acceptance:$run_id:%' AND released_at IS NULL AND expires_at>now()),
|
||||
'activeRateReservations',(SELECT count(*) FROM gateway_rate_limit_reservations WHERE scope_key LIKE 'acceptance:$run_id:%' AND status='reserved'),
|
||||
'activeRiverJobs',(SELECT count(*) FROM river_job job JOIN gateway_tasks task ON task.river_job_id=job.id WHERE task.acceptance_run_id='$run_id'::uuid AND job.state IN ('available','pending','retryable','running','scheduled'))
|
||||
);")
|
||||
duplicates=$(database_query "
|
||||
SELECT jsonb_build_object(
|
||||
'remoteTaskIds',(SELECT count(*) FROM (SELECT remote_task_id FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid AND remote_task_id IS NOT NULL GROUP BY remote_task_id HAVING count(*)>1) duplicate),
|
||||
'billingTransactions',(SELECT count(*) FROM (SELECT reference_id,transaction_type FROM gateway_wallet_transactions WHERE reference_type='gateway_task' AND reference_id IN (SELECT id::text FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid) GROUP BY reference_id,transaction_type HAVING count(*)>1) duplicate)
|
||||
);")
|
||||
sample_summary=$(jq -s '[.[].workers[]] | group_by(.instanceId) | map({
|
||||
instanceId:.[0].instanceId,
|
||||
peakActiveTasks:(map(.activeTasks)|max),
|
||||
peakPreparingTasks:(map(.preparingTasks)|max),
|
||||
peakWaitingUpstreamTasks:(map(.waitingUpstreamTasks)|max),
|
||||
peakFinalizingTasks:(map(.finalizingTasks)|max),
|
||||
minSafeCapacity:(map(.safeCapacity)|min),
|
||||
maxSafeCapacity:(map(.safeCapacity)|max),
|
||||
pressureStates:(map(.pressureState)|unique)
|
||||
})' "$samples")
|
||||
resource_summary=$(jq -Rn '
|
||||
def number_without($suffix): rtrimstr($suffix) | tonumber;
|
||||
def memory_mib:
|
||||
if endswith("Gi") then number_without("Gi") * 1024
|
||||
elif endswith("Mi") then number_without("Mi")
|
||||
elif endswith("Ki") then number_without("Ki") / 1024
|
||||
else tonumber end;
|
||||
[inputs | split(",") | select(.[0] != "timestamp") | {
|
||||
scope:.[1],name:.[2],cpu:.[3],memory:.[4]
|
||||
}]
|
||||
| group_by([.scope,.name])
|
||||
| map(if .[0].scope == "node" then {
|
||||
scope:"node",name:.[0].name,
|
||||
maxCpuPercent:(map(.cpu | number_without("%")) | max),
|
||||
maxMemoryPercent:(map(.memory | number_without("%")) | max)
|
||||
} else {
|
||||
scope:"pod",name:.[0].name,
|
||||
maxCpuMillicores:(map(.cpu | if endswith("m") then number_without("m") else tonumber * 1000 end) | max),
|
||||
maxMemoryMiB:(map(.memory | memory_mib) | max)
|
||||
} end)' <"$resources")
|
||||
phases=$(jq -s '[.[] | .phases[]]' "$report_root"/adaptive-load-*.json)
|
||||
local passed=true bottleneck=configured_probe_ceiling
|
||||
if ((highest < 1)); then passed=false; fi
|
||||
if ((failed > 0)); then bottleneck=worker_or_gateway_limit; fi
|
||||
if jq -e 'any(.[]; .etcdHealthy == false)' <<<"$levels" >/dev/null; then
|
||||
bottleneck=local_control_plane_limit
|
||||
fi
|
||||
if ! jq -e 'length == 3' <<<"$workers" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'all(.[]; ((.used + .reserved) <= .limit))' <<<"$limits" >/dev/null; then passed=false; fi
|
||||
if ! jq -e 'all(.[]; (.limit == null or .peak <= .limit))' <<<"$concurrency_peaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.activeConcurrencyLeases==0 and .activeRateReservations==0 and .activeRiverJobs==0' <<<"$leaks" >/dev/null; then passed=false; fi
|
||||
if ! jq -e '.remoteTaskIds==0 and .billingTransactions==0' <<<"$duplicates" >/dev/null; then passed=false; fi
|
||||
jq -n \
|
||||
--arg runId "$run_id" --arg startedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \
|
||||
--arg bottleneck "$bottleneck" --argjson passed "$passed" \
|
||||
--argjson highest "$highest" --argjson failed "$failed" \
|
||||
--argjson levels "$levels" --argjson phases "$phases" \
|
||||
--argjson workers "$workers" --argjson distribution "$distribution" \
|
||||
--argjson routing "$routing" --argjson limits "$limits" \
|
||||
--argjson concurrencyPeaks "$concurrency_peaks" --argjson workerPeaks "$sample_summary" \
|
||||
--argjson resourcePeaks "$resource_summary" \
|
||||
--argjson leaks "$leaks" --argjson duplicates "$duplicates" \
|
||||
'{
|
||||
schemaVersion:"acceptance-load-report/v2",runId:$runId,profile:"adaptive-capacity",
|
||||
startedAt:$startedAt,finishedAt:(now|todateiso8601),passed:$passed,secretSafe:true,
|
||||
phases:$phases,
|
||||
capacity:{levels:$levels,highestStableConcurrency:$highest,firstFailedConcurrency:$failed,bottleneck:$bottleneck},
|
||||
cluster:{workers:$workers,workerPeaks:$workerPeaks,taskDistribution:$distribution,
|
||||
platformRouting:$routing,rateLimits:$limits,concurrencyPeaks:$concurrencyPeaks,
|
||||
resourcePeaks:$resourcePeaks,leaks:$leaks,duplicates:$duplicates}
|
||||
}' >"$output"
|
||||
chmod 0600 "$output"
|
||||
jq -e '.passed==true and .secretSafe==true' "$output" >/dev/null
|
||||
}
|
||||
|
||||
sample_resources() {
|
||||
local output=$1 stop_file=$2
|
||||
printf 'timestamp,scope,name,cpu,memory\n' >"$output"
|
||||
@@ -567,6 +865,64 @@ quick() {
|
||||
echo "local_acceptance_quick=PASS run_id=$run_id report=$report_root/quick.json"
|
||||
}
|
||||
|
||||
adaptive() {
|
||||
local hard_limit=${AI_GATEWAY_LOCAL_ADAPTIVE_HARD_LIMIT:-32}
|
||||
local max_probe=${AI_GATEWAY_LOCAL_ADAPTIVE_MAX_CONCURRENCY:-$((hard_limit * 3))}
|
||||
local level=4 highest=0 failed=0 low high mid
|
||||
local samples="$report_root/adaptive-worker-samples.ndjson"
|
||||
local resources="$report_root/adaptive-resources.csv"
|
||||
worker_sample_stop="$report_root/adaptive-worker-samples.stop"
|
||||
[[ ! -e $samples && ! -e $resources && ! -e $worker_sample_stop ]] || fail_gate acceptance_report_exists "adaptive evidence already exists"
|
||||
current_phase=adaptive_workers
|
||||
configure_adaptive_workers
|
||||
sample_worker_runtime "$samples" "$worker_sample_stop" &
|
||||
active_worker_sampler_pid=$!
|
||||
sample_resources "$resources" "$worker_sample_stop" &
|
||||
active_resource_sampler_pid=$!
|
||||
while ((level <= max_probe)); do
|
||||
current_phase="adaptive_capacity_$level"
|
||||
if run_adaptive_level "$level"; then
|
||||
highest=$level
|
||||
level=$((level * 2))
|
||||
else
|
||||
failed=$level
|
||||
break
|
||||
fi
|
||||
done
|
||||
if ((failed > 0 && highest > 0)); then
|
||||
low=$((highest + 1))
|
||||
high=$((failed - 1))
|
||||
while ((low <= high)); do
|
||||
mid=$(((low + high) / 2))
|
||||
current_phase="adaptive_capacity_refine_$mid"
|
||||
if run_adaptive_level "$mid"; then
|
||||
highest=$mid
|
||||
low=$((mid + 1))
|
||||
else
|
||||
failed=$mid
|
||||
high=$((mid - 1))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
touch "$worker_sample_stop"
|
||||
wait "$active_worker_sampler_pid"
|
||||
wait "$active_resource_sampler_pid"
|
||||
active_worker_sampler_pid=
|
||||
active_resource_sampler_pid=
|
||||
worker_sample_stop=
|
||||
local ready_workers registered_workers
|
||||
ready_workers=$(kubectl --context "$context" -n "$namespace" get pods \
|
||||
-l app.kubernetes.io/name=easyai-worker -o json |
|
||||
jq '[.items[] | select(.status.phase=="Running" and any(.status.containerStatuses[]?; .ready==true))] | length')
|
||||
registered_workers=$(database_query "SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now() - interval '30 seconds';")
|
||||
[[ $ready_workers == 3 && $registered_workers == 3 ]] ||
|
||||
fail_gate adaptive_worker_count "adaptive validation requires three Ready and registered Workers throughout the final gate"
|
||||
verify_hard_gates
|
||||
current_phase=adaptive_report
|
||||
build_adaptive_report "$report_root/adaptive-capacity.json" "$samples" "$resources" "$highest" "$failed"
|
||||
echo "local_acceptance_adaptive=PASS run_id=$run_id highest_stable_concurrency=$highest first_failed_concurrency=$failed report=$report_root/adaptive-capacity.json"
|
||||
}
|
||||
|
||||
full() {
|
||||
local manifest=$1 profile repetition
|
||||
quick
|
||||
@@ -610,7 +966,7 @@ full() {
|
||||
command=${1:-}
|
||||
shift || true
|
||||
case $command in
|
||||
quick)
|
||||
quick|adaptive|provider-burst)
|
||||
[[ $# -eq 0 ]] || { usage >&2; exit 64; }
|
||||
;;
|
||||
artifact-smoke|full)
|
||||
@@ -628,6 +984,12 @@ case $command in
|
||||
quick)
|
||||
quick
|
||||
;;
|
||||
adaptive)
|
||||
adaptive
|
||||
;;
|
||||
provider-burst)
|
||||
provider_burst
|
||||
;;
|
||||
artifact-smoke)
|
||||
artifact_smoke "$2"
|
||||
;;
|
||||
|
||||
@@ -15,6 +15,8 @@ Usage:
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--execute dist/releases/<SHA>.json \
|
||||
--skip-local-acceptance
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--execute-single-node dist/releases/<SHA>.json
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--promote dist/releases/<SHA>.json --run-id <production-run-id>
|
||||
|
||||
@@ -29,6 +31,10 @@ The explicit --skip-local-acceptance form records both local stages as skipped
|
||||
with a user-directed waiver. It never records them as passed and preserves all
|
||||
online simulation, real canary, resource, consistency, and release CAS gates.
|
||||
|
||||
The --execute-single-node form is a Ningbo-only diagnostic. It keeps production
|
||||
traffic in validation, routes acceptance candidates only to the protocol
|
||||
emulator, never runs real-canary, and leaves the Run non-promotable.
|
||||
|
||||
Required private environment values:
|
||||
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN, or AI_GATEWAY_ONLINE_ACCOUNT/PASSWORD
|
||||
|
||||
@@ -54,6 +60,7 @@ if [[ ${1:-} == --promote ]]; then
|
||||
fi
|
||||
|
||||
skip_local_acceptance=false
|
||||
single_node_acceptance=false
|
||||
local_acceptance_report=
|
||||
if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
|
||||
release_manifest=$2
|
||||
@@ -61,6 +68,10 @@ if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
|
||||
elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then
|
||||
release_manifest=$2
|
||||
skip_local_acceptance=true
|
||||
elif [[ ${1:-} == --execute-single-node && $# -eq 2 ]]; then
|
||||
release_manifest=$2
|
||||
skip_local_acceptance=true
|
||||
single_node_acceptance=true
|
||||
else
|
||||
usage >&2
|
||||
exit 64
|
||||
@@ -79,6 +90,18 @@ fi
|
||||
load_cluster_env
|
||||
require_commands curl git go jq node openssl sed shasum
|
||||
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG=0
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG=0
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO=1
|
||||
AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG=0
|
||||
AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS=${AI_GATEWAY_ACCEPTANCE_IDENTITY_SHARDS:-16}
|
||||
AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB=512
|
||||
AI_GATEWAY_ACCEPTANCE_DATABASE_MAX_CONN_IDLE_SECONDS=30
|
||||
fi
|
||||
|
||||
: "${AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_API_KEY:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_API_KEY_ID:=}"
|
||||
@@ -109,15 +132,45 @@ require_commands curl git go jq node openssl sed shasum
|
||||
: "${AI_GATEWAY_ACCEPTANCE_OVERLOAD_DURATION:=10m}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_GATEWAYS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS:=8}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS:=96}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS:=8 12 16 24 32 40 48}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_REAL_IMAGE_URLS:=}"
|
||||
: "${AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL:=}"
|
||||
: "${AI_GATEWAY_ONLINE_ACCOUNT:=}"
|
||||
: "${AI_GATEWAY_ONLINE_PASSWORD:=}"
|
||||
: "${AI_GATEWAY_ONLINE_BASE_URL:=}"
|
||||
if [[ -z $AI_GATEWAY_ACCEPTANCE_GATEWAYS ]]; then
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
|
||||
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS='http://10.77.0.1:18089'
|
||||
else
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAYS="https://${CLUSTER_NINGBO_HOST#root@},https://${CLUSTER_HONGKONG_HOST#root@}"
|
||||
: "${AI_GATEWAY_DEPLOY_DOMAIN:?}"
|
||||
AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME=${AI_GATEWAY_ACCEPTANCE_GATEWAY_TLS_SERVER_NAME:-$AI_GATEWAY_DEPLOY_DOMAIN}
|
||||
fi
|
||||
fi
|
||||
single_node_video_slots=()
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS -le 128 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS -le 128 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS =~ ^[1-9][0-9]*$ ]] || {
|
||||
echo 'single-node baseline, GEMINI slots, and GEMINI requests must be positive bounded integers' >&2
|
||||
exit 1
|
||||
}
|
||||
read -r -a single_node_video_slots <<<"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS"
|
||||
(( ${#single_node_video_slots[@]} > 0 )) || {
|
||||
echo 'AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_VIDEO_SLOTS must contain at least one slot count' >&2
|
||||
exit 1
|
||||
}
|
||||
for single_node_slot in "${single_node_video_slots[@]}"; do
|
||||
[[ $single_node_slot =~ ^[1-9][0-9]*$ && $single_node_slot -le 128 ]] || {
|
||||
echo 'single-node video slot counts must be integers between 1 and 128' >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
fi
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS =~ ^[1-9][0-9]*$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_API_DATABASE_MAX_CONNS -le 256 ]] || {
|
||||
@@ -189,16 +242,27 @@ fi
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO =~ ^[0-9]+$ &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG =~ ^[0-9]+$ &&
|
||||
$((AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG)) -eq 2 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MIN_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG -le $AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_NINGBO -le 16 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_AUTOSCALING_MAX_REPLICAS_HONGKONG -le 16 ]] || {
|
||||
echo 'acceptance requires exactly two baseline Workers and ordered 0..16 per-site autoscaling bounds' >&2
|
||||
echo 'acceptance baseline Worker topology or per-site autoscaling bounds are invalid' >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
[[ $AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO == 1 &&
|
||||
$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 0 ]] || {
|
||||
echo 'single-node acceptance requires one Ningbo Worker and zero Hong Kong Workers' >&2
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
(( AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO + AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG == 2 )) || {
|
||||
echo 'production acceptance requires exactly two baseline Workers' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
node "$cluster_root/scripts/release-manifest.mjs" validate "$release_manifest" >/dev/null
|
||||
release_sha=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_manifest" sourceSha)
|
||||
@@ -223,24 +287,30 @@ if [[ $skip_local_acceptance != true ]]; then
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
[[ -z $(git -C "$cluster_root" status --short) ]] || {
|
||||
echo 'production acceptance requires a clean release working copy' >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ $single_node_acceptance != true ]]; then
|
||||
[[ -z $(git -C "$cluster_root" status --short) ]] || {
|
||||
echo 'production acceptance requires a clean release working copy' >&2
|
||||
exit 1
|
||||
}
|
||||
else
|
||||
echo 'single_node_acceptance_tool=working_tree diagnostic_only=true certification=false'
|
||||
fi
|
||||
acceptance_tool_sha=$(git -C "$cluster_root" rev-parse HEAD)
|
||||
if [[ $acceptance_tool_sha != "$release_sha" ]]; then
|
||||
git -C "$cluster_root" merge-base --is-ancestor "$release_sha" "$acceptance_tool_sha" || {
|
||||
echo 'production release must be an ancestor of the acceptance tool HEAD' >&2
|
||||
exit 1
|
||||
}
|
||||
acceptance_tool_delta=$(node "$cluster_root/scripts/release-components.mjs" \
|
||||
"$release_sha" "$acceptance_tool_sha")
|
||||
[[ $(jq -r '.components' <<<"$acceptance_tool_delta") == none &&
|
||||
$(jq -r '.migrationsChanged' <<<"$acceptance_tool_delta") == false ]] || {
|
||||
echo 'acceptance tool HEAD contains runtime or migration changes beyond the production release' >&2
|
||||
exit 1
|
||||
}
|
||||
echo "acceptance_tool_delta=PASS release=$release_sha tool_sha=$acceptance_tool_sha runtime_changes=false"
|
||||
if [[ $single_node_acceptance != true ]]; then
|
||||
acceptance_tool_delta=$(node "$cluster_root/scripts/release-components.mjs" \
|
||||
"$release_sha" "$acceptance_tool_sha")
|
||||
[[ $(jq -r '.components' <<<"$acceptance_tool_delta") == none &&
|
||||
$(jq -r '.migrationsChanged' <<<"$acceptance_tool_delta") == false ]] || {
|
||||
echo 'acceptance tool HEAD contains runtime or migration changes beyond the production release' >&2
|
||||
exit 1
|
||||
}
|
||||
echo "acceptance_tool_delta=PASS release=$release_sha tool_sha=$acceptance_tool_sha runtime_changes=false"
|
||||
fi
|
||||
fi
|
||||
|
||||
namespace=${AI_GATEWAY_K3S_NAMESPACE:-easyai}
|
||||
@@ -262,6 +332,7 @@ run_id=
|
||||
report_root=
|
||||
stable_profile=P24
|
||||
active_profile=P24
|
||||
single_node_stable_slots=4
|
||||
failure_reason=
|
||||
failure_gate_id=
|
||||
failure_recorded=false
|
||||
@@ -272,6 +343,7 @@ acceptance_participants_json='[]'
|
||||
AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
|
||||
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
|
||||
acceptance_load_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-load-linux-amd64
|
||||
acceptance_emulator_linux_binary=$temporary_root/easyai-ai-gateway-acceptance-emulator-linux-amd64
|
||||
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
|
||||
current_production_snapshot=$temporary_root/current-production-snapshot.json
|
||||
local_snapshot_config_hash=
|
||||
@@ -291,6 +363,8 @@ video_admitted_throughput=
|
||||
certified_max_replicas_ningbo=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_NINGBO
|
||||
certified_max_replicas_hongkong=$AI_GATEWAY_ACCEPTANCE_BASE_REPLICAS_HONGKONG
|
||||
runtime_observation_started_at=
|
||||
single_node_emulator_url=
|
||||
single_node_emulator_installed=false
|
||||
|
||||
cleanup() {
|
||||
local status=$?
|
||||
@@ -306,9 +380,16 @@ cleanup() {
|
||||
if [[ $remote_load_drivers_installed == true ]]; then
|
||||
cleanup_remote_load_drivers >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [[ $single_node_emulator_installed == true ]]; then
|
||||
cleanup_single_node_emulator >/dev/null 2>&1 || true
|
||||
fi
|
||||
if (( status != 0 )) && [[ -n $run_id && $failure_recorded != true ]]; then
|
||||
set +e
|
||||
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
apply_single_node_capacity "$single_node_stable_slots" >/dev/null 2>&1
|
||||
else
|
||||
apply_capacity_profile "$stable_profile" >/dev/null 2>&1
|
||||
fi
|
||||
[[ -n $failure_gate_id ]] || failure_gate_id=workflow_unexpected_exit
|
||||
mark_run_failed "${failure_reason:-acceptance workflow exited unexpectedly}"
|
||||
set -e
|
||||
@@ -361,6 +442,10 @@ bootstrap_acceptance_admin_token
|
||||
-trimpath -o "$acceptance_load_binary" ./cmd/acceptance-load
|
||||
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-trimpath -o "$acceptance_load_linux_binary" ./cmd/acceptance-load
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
env -u AI_GATEWAY_TEST_DATABASE_URL CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-trimpath -o "$acceptance_emulator_linux_binary" ./cmd/acceptance-emulator
|
||||
fi
|
||||
env -u AI_GATEWAY_TEST_DATABASE_URL go build \
|
||||
-trimpath -o "$acceptance_snapshot_binary" ./cmd/acceptance-snapshot
|
||||
)
|
||||
@@ -1386,14 +1471,93 @@ WHERE NOT EXISTS (
|
||||
echo "acceptance_model_access=PASS resources=$expected"
|
||||
}
|
||||
|
||||
cleanup_single_node_emulator() {
|
||||
[[ $run_token =~ ^[0-9a-f]{64}$ ]] || return 0
|
||||
local suffix=${run_token:0:12}
|
||||
local binary_path=/root/easyai-acceptance-emulator-$suffix
|
||||
local pid_path=$binary_path.pid
|
||||
local log_path=$binary_path.log
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE' || true
|
||||
set -euo pipefail
|
||||
binary_path=$1
|
||||
pid_path=$2
|
||||
log_path=$3
|
||||
if [[ -f $pid_path && ! -L $pid_path ]]; then
|
||||
pid=$(<"$pid_path")
|
||||
if [[ $pid =~ ^[0-9]+$ && -e /proc/$pid/exe && $(readlink -f "/proc/$pid/exe") == "$binary_path" ]]; then
|
||||
kill -TERM "$pid"
|
||||
for _ in {1..20}; do
|
||||
kill -0 "$pid" >/dev/null 2>&1 || break
|
||||
sleep 0.1
|
||||
done
|
||||
fi
|
||||
fi
|
||||
for path in "$binary_path" "$pid_path" "$log_path"; do
|
||||
[[ ! -e $path ]] || unlink "$path"
|
||||
done
|
||||
REMOTE
|
||||
single_node_emulator_installed=false
|
||||
}
|
||||
|
||||
start_single_node_emulator() {
|
||||
[[ $run_token =~ ^[0-9a-f]{64}$ ]]
|
||||
local suffix=${run_token:0:12}
|
||||
local binary_path=/root/easyai-acceptance-emulator-$suffix
|
||||
local pid_path=$binary_path.pid
|
||||
local log_path=$binary_path.log
|
||||
local local_digest remote_digest node_ip
|
||||
local_digest=$(shasum -a 256 "$acceptance_emulator_linux_binary" | awk '{print $1}')
|
||||
cluster_scp "$acceptance_emulator_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
|
||||
remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" "chmod 0755 '$binary_path'; sha256sum '$binary_path'" | awk '{print $1}')
|
||||
[[ $remote_digest == "$local_digest" ]]
|
||||
single_node_emulator_installed=true
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- "$binary_path" "$pid_path" "$log_path" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
binary_path=$1
|
||||
pid_path=$2
|
||||
log_path=$3
|
||||
if ss -ltn '( sport = :18090 )' | tail -n +2 | grep -q .; then
|
||||
echo 'single-node emulator port 18090 is already in use' >&2
|
||||
exit 1
|
||||
fi
|
||||
nohup env HTTP_ADDR=:18090 "$binary_path" </dev/null >"$log_path" 2>&1 &
|
||||
pid=$!
|
||||
printf '%s\n' "$pid" >"$pid_path"
|
||||
chmod 0600 "$pid_path" "$log_path"
|
||||
for _ in {1..50}; do
|
||||
if curl -fsS --max-time 2 http://127.0.0.1:18090/healthz >/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
kill -0 "$pid" >/dev/null 2>&1 || exit 1
|
||||
sleep 0.2
|
||||
done
|
||||
exit 1
|
||||
REMOTE
|
||||
node_ip=$(remote_kubectl get node easyai-ningbo -o 'jsonpath={.status.addresses[?(@.type=="InternalIP")].address}')
|
||||
[[ $node_ip =~ ^[0-9a-fA-F:.]+$ ]]
|
||||
single_node_emulator_url=http://$node_ip:18090
|
||||
remote_kubectl exec -n "$namespace" deployment/easyai-api-ningbo -- \
|
||||
wget -qO- "$single_node_emulator_url/healthz" >/dev/null
|
||||
echo 'single_node_protocol_emulator=PASS placement=ningbo-host digest_verified=true pod_reachable=true'
|
||||
}
|
||||
|
||||
deploy_protocol_emulator() {
|
||||
sed "s|image: easyai-api|image: $api_image|" \
|
||||
"$cluster_root/deploy/kubernetes/acceptance/protocol-emulator.yaml" |
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" 'k3s kubectl apply -f -' >/dev/null
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
remote_kubectl patch deployment easyai-acceptance-emulator -n "$namespace" \
|
||||
--type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
|
||||
remote_kubectl patch deployment easyai-acceptance-callback-collector -n "$namespace" \
|
||||
--type=json -p='[{"op":"replace","path":"/spec/template/spec/nodeSelector","value":{"kubernetes.io/hostname":"easyai-ningbo"}}]' >/dev/null
|
||||
fi
|
||||
remote_kubectl rollout status deployment/easyai-acceptance-emulator \
|
||||
-n "$namespace" --timeout=300s
|
||||
remote_kubectl rollout status deployment/easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --timeout=300s
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
start_single_node_emulator
|
||||
fi
|
||||
}
|
||||
|
||||
is_release_ancestor() {
|
||||
@@ -1408,7 +1572,11 @@ create_and_activate_run() {
|
||||
local activate_response=$temporary_root/activate-run.json
|
||||
local traffic_response=$temporary_root/pre-activate-traffic.json
|
||||
local previous_run_response=$temporary_root/previous-run.json
|
||||
local body
|
||||
local body emulator_url='http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090'
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
[[ $single_node_emulator_url == http://* ]]
|
||||
emulator_url=$single_node_emulator_url
|
||||
fi
|
||||
body=$(jq -cn \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg apiDigest "$api_digest" \
|
||||
@@ -1416,7 +1584,7 @@ create_and_activate_run() {
|
||||
--arg apiKeyID "$AI_GATEWAY_ACCEPTANCE_API_KEY_ID" \
|
||||
--arg userID "$AI_GATEWAY_ACCEPTANCE_USER_ID" \
|
||||
--arg token "$run_token" \
|
||||
--arg emulatorURL 'http://easyai-acceptance-emulator.easyai.svc.cluster.local:8090' \
|
||||
--arg emulatorURL "$emulator_url" \
|
||||
--arg callbackURL 'http://easyai-acceptance-callback-collector.easyai.svc.cluster.local:8091/callbacks' \
|
||||
--argjson participants "$acceptance_participants_json" \
|
||||
'{
|
||||
@@ -2271,8 +2439,10 @@ cleanup_remote_load_drivers() {
|
||||
hongkong_env=$(remote_load_env_path hongkong)
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$ningbo_env' ]] || unlink '$ningbo_env'" || true
|
||||
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
|
||||
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
|
||||
if [[ $single_node_acceptance != true ]]; then
|
||||
cluster_ssh "$CLUSTER_HONGKONG_HOST" \
|
||||
"pkill -TERM -f '^$binary_path( |$)' >/dev/null 2>&1 || true; [[ ! -e '$binary_path' ]] || unlink '$binary_path'; [[ ! -e '$hongkong_env' ]] || unlink '$hongkong_env'" || true
|
||||
fi
|
||||
remote_load_drivers_installed=false
|
||||
}
|
||||
|
||||
@@ -3268,6 +3438,363 @@ certify_worker_resource_requests() {
|
||||
}' >"$report_root/certified-worker-resources.json"
|
||||
}
|
||||
|
||||
apply_single_node_capacity() {
|
||||
local slots=$1
|
||||
[[ $slots =~ ^[0-9]+$ ]] && (( slots >= 4 && slots <= 48 )) || return 1
|
||||
local worker_pool=$((slots + 12))
|
||||
local target_outstanding=$((slots * 2))
|
||||
local config_patch
|
||||
config_patch=$(jq -nc \
|
||||
--arg slots "$slots" \
|
||||
--arg workerPool "$worker_pool" \
|
||||
--arg targetOutstanding "$target_outstanding" \
|
||||
'{data:{
|
||||
AI_GATEWAY_WORKER_REPLICAS_NINGBO:"1",
|
||||
AI_GATEWAY_WORKER_REPLICAS_HONGKONG:"0",
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO:"1",
|
||||
AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG:"0",
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO:"1",
|
||||
AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG:"0",
|
||||
AI_GATEWAY_WORKER_AUTOSCALING_ENABLED:"false",
|
||||
AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT:$slots,
|
||||
AI_GATEWAY_WORKER_DATABASE_MAX_CONNS:$workerPool,
|
||||
AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY:$slots,
|
||||
AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY:$slots,
|
||||
AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA:$targetOutstanding
|
||||
}}')
|
||||
remote_kubectl label node easyai-ningbo easyai.io/worker=true --overwrite >/dev/null
|
||||
remote_kubectl scale deployment/easyai-capacity-controller -n "$namespace" --replicas=0 >/dev/null
|
||||
remote_kubectl scale deployment/easyai-worker-hongkong -n "$namespace" --replicas=0 >/dev/null
|
||||
remote_kubectl patch configmap easyai-ai-gateway-config -n "$namespace" \
|
||||
--type=merge -p "$config_patch" >/dev/null
|
||||
remote_kubectl set env deployment/easyai-worker-ningbo -n "$namespace" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT=$slots" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" \
|
||||
"AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE=$AI_GATEWAY_ACCEPTANCE_ASYNC_ADMISSION_MICROBATCH_SIZE" \
|
||||
"AI_GATEWAY_DATABASE_MAX_CONNS=$worker_pool" \
|
||||
'AI_GATEWAY_DATABASE_CRITICAL_MAX_CONNS=4' \
|
||||
"AI_GATEWAY_DATABASE_RIVER_MAX_CONNS=$AI_GATEWAY_ACCEPTANCE_WORKER_RIVER_MAX_CONNS" \
|
||||
'AI_GATEWAY_DATABASE_MIN_IDLE_CONNS=4' \
|
||||
'AI_GATEWAY_DATABASE_MAX_CONN_IDLE_SECONDS=30' \
|
||||
"AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY=$slots" \
|
||||
"AI_GATEWAY_MEDIA_REQUEST_CONCURRENCY=$slots" \
|
||||
'AI_GATEWAY_MEDIA_IMAGE_NORMALIZATION_CONCURRENCY=2' >/dev/null
|
||||
remote_kubectl set resources deployment/easyai-worker-ningbo -n "$namespace" \
|
||||
--containers=worker \
|
||||
--requests="cpu=${AI_GATEWAY_ACCEPTANCE_WORKER_CPU_REQUEST_MILLICORES}m,memory=${AI_GATEWAY_ACCEPTANCE_WORKER_MEMORY_REQUEST_MIB}Mi" \
|
||||
--limits='cpu=2,memory=2Gi' >/dev/null
|
||||
remote_kubectl scale deployment/easyai-worker-ningbo -n "$namespace" --replicas=1 >/dev/null
|
||||
remote_kubectl rollout status deployment/easyai-worker-ningbo -n "$namespace" --timeout=300s
|
||||
remote_kubectl set env deployment/easyai-api-ningbo -n "$namespace" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT=$slots" \
|
||||
"AI_GATEWAY_ASYNC_WORKER_GLOBAL_HARD_LIMIT=$slots" >/dev/null
|
||||
remote_kubectl rollout status deployment/easyai-api-ningbo -n "$namespace" --timeout=300s
|
||||
|
||||
local deadline=$((SECONDS + 90)) allocation
|
||||
while (( SECONDS < deadline )); do
|
||||
allocation=$(database_query "
|
||||
SELECT count(*)||':'||COALESCE(max(allocated_capacity),0)
|
||||
FROM gateway_worker_instances
|
||||
WHERE status='active' AND heartbeat_at > now()-interval '30 seconds';")
|
||||
if [[ $allocation == "1:$slots" ]]; then
|
||||
local metrics_ready=false
|
||||
for _ in {1..30}; do
|
||||
if remote_kubectl top node easyai-ningbo --no-headers >/dev/null 2>&1 &&
|
||||
[[ -n $(remote_kubectl top pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' \
|
||||
--no-headers 2>/dev/null) ]]; then
|
||||
metrics_ready=true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
[[ $metrics_ready == true ]] || {
|
||||
echo 'single-node metrics did not become ready after Worker rollout' >&2
|
||||
return 1
|
||||
}
|
||||
active_profile=S$slots
|
||||
echo "single_node_capacity=PASS slots=$slots worker_pool=$worker_pool active_instances=1"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "single-node Worker allocation did not converge: expected=1:$slots actual=${allocation:-missing}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
install_single_node_load_driver() {
|
||||
[[ $remote_load_drivers_installed == false ]] || return 0
|
||||
local binary_path env_file remote_env local_digest remote_digest
|
||||
binary_path=$(remote_load_binary_path)
|
||||
remote_env=$(remote_load_env_path ningbo)
|
||||
env_file=$temporary_root/remote-load-ningbo.env
|
||||
write_remote_load_env "$env_file" "$AI_GATEWAY_ACCEPTANCE_GATEWAYS"
|
||||
local_digest=$(shasum -a 256 "$acceptance_load_linux_binary" | awk '{print $1}')
|
||||
cluster_scp "$acceptance_load_linux_binary" "$CLUSTER_NINGBO_HOST:$binary_path" >/dev/null
|
||||
cluster_scp "$env_file" "$CLUSTER_NINGBO_HOST:$remote_env" >/dev/null
|
||||
remote_digest=$(cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
"chmod 0755 '$binary_path'; chmod 0600 '$remote_env'; sha256sum '$binary_path'" | awk '{print $1}')
|
||||
[[ $remote_digest == "$local_digest" ]]
|
||||
remote_load_drivers_installed=true
|
||||
echo 'acceptance_load_driver=PASS sites=1 placement=ningbo-host digest_verified=true'
|
||||
}
|
||||
|
||||
run_single_node_load_profile() {
|
||||
local profile=$1
|
||||
local report_path=$2
|
||||
local requests=$3
|
||||
install_single_node_load_driver
|
||||
local binary_path remote_env artifact remote_report status=0
|
||||
binary_path=$(remote_load_binary_path)
|
||||
remote_env=$(remote_load_env_path ningbo)
|
||||
artifact=$(basename "$report_path")
|
||||
remote_report=/root/easyai-acceptance-load-"$run_id"-"$artifact"
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" bash -s -- \
|
||||
"$binary_path" "$remote_env" "$profile" "$remote_report" "${artifact%.json}" "$requests" \
|
||||
>"$temporary_root/$artifact.stdout" <<'REMOTE' || status=$?
|
||||
set -euo pipefail
|
||||
binary_path=$1
|
||||
env_file=$2
|
||||
profile=$3
|
||||
report=$4
|
||||
execution_id=$5
|
||||
requests=$6
|
||||
[[ ! -e $report ]] || unlink "$report"
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "$env_file"
|
||||
set +a
|
||||
"$binary_path" -profile "$profile" -report "$report" \
|
||||
-shard-index 0 -shard-count 1 -execution-id "$execution_id" \
|
||||
-requests "$requests"
|
||||
REMOTE
|
||||
cluster_scp "$CLUSTER_NINGBO_HOST:$remote_report" "$report_path" >/dev/null || return 1
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" "[[ ! -e '$remote_report' ]] || unlink '$remote_report'" >/dev/null || true
|
||||
chmod 0600 "$report_path"
|
||||
jq -e --arg runId "$run_id" --arg profile "$profile" \
|
||||
'.schemaVersion == "acceptance-load-report/v1" and .runId == $runId and .profile == $profile and .secretSafe == true' \
|
||||
"$report_path" >/dev/null || return 1
|
||||
(( status == 0 )) && jq -e '.passed == true' "$report_path" >/dev/null
|
||||
}
|
||||
|
||||
sample_single_node_pressure() {
|
||||
local output=$1
|
||||
local stop_file=$2
|
||||
local failure_file=$3
|
||||
echo 'timestamp,queued,running,db_connections,db_max_connections,active_instances,allocated_capacity,node_memory_percent,worker_memory_mib,worker_cpu_millicores,restarts' >"$output"
|
||||
while [[ ! -f $stop_file ]]; do
|
||||
local state node_memory worker_resources restarts row
|
||||
state=$(database_query "
|
||||
SELECT
|
||||
count(*) FILTER (WHERE status='queued')||','||
|
||||
count(*) FILTER (WHERE status='running')||','||
|
||||
(SELECT count(*) FROM pg_stat_activity WHERE backend_type='client backend')||','||
|
||||
(SELECT setting FROM pg_settings WHERE name='max_connections')||','||
|
||||
(SELECT count(*) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')||','||
|
||||
(SELECT COALESCE(max(allocated_capacity),0) FROM gateway_worker_instances WHERE status='active' AND heartbeat_at > now()-interval '30 seconds')
|
||||
FROM gateway_tasks
|
||||
WHERE acceptance_run_id='$run_id'::uuid;") || return 1
|
||||
node_memory=$(remote_kubectl top node easyai-ningbo --no-headers |
|
||||
awk '{value=$5; sub(/%$/, "", value); print value}') || return 1
|
||||
worker_resources=$(remote_kubectl top pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo' --no-headers |
|
||||
awk '{
|
||||
cpu=$2; memory=$3
|
||||
if (cpu ~ /n$/) {sub(/n$/, "", cpu); cpu/=1000000}
|
||||
else if (cpu ~ /u$/) {sub(/u$/, "", cpu); cpu/=1000}
|
||||
else if (cpu ~ /m$/) {sub(/m$/, "", cpu)}
|
||||
else {cpu*=1000}
|
||||
if (memory ~ /Gi$/) {sub(/Gi$/, "", memory); memory*=1024}
|
||||
else if (memory ~ /Mi$/) {sub(/Mi$/, "", memory)}
|
||||
else if (memory ~ /Ki$/) {sub(/Ki$/, "", memory); memory/=1024}
|
||||
print int(memory+0) "," int(cpu+0)
|
||||
}') || return 1
|
||||
restarts=$(remote_kubectl get pods -n "$namespace" \
|
||||
-l 'app.kubernetes.io/part-of=easyai-ai-gateway' -o json |
|
||||
jq '[.items[].status.containerStatuses[]?.restartCount] | add // 0') || return 1
|
||||
row="$(date -u '+%Y-%m-%dT%H:%M:%SZ'),$state,$node_memory,$worker_resources,$restarts"
|
||||
printf '%s\n' "$row" >>"$output"
|
||||
if ! awk -F',' '($5<=0)||($4*4>=$5*3)||($6!=1)||($7<1)||($8>=85)||($9>=1536)||($11>0){exit 1}' <<<"$row"; then
|
||||
printf '%s\n' "$row" >"$failure_file"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
record_single_node_result() {
|
||||
local workload=$1
|
||||
local slots=$2
|
||||
local load_report=$3
|
||||
local pressure_report=$4
|
||||
local summary=$5
|
||||
local peaks
|
||||
peaks=$(awk -F',' '
|
||||
NR>1 {
|
||||
samples++
|
||||
if ($2>queued) queued=$2
|
||||
if ($3>running) running=$3
|
||||
if ($4>db) db=$4
|
||||
if ($7>allocated) allocated=$7
|
||||
if ($8>node_memory) node_memory=$8
|
||||
if ($9>worker_memory) worker_memory=$9
|
||||
if ($10>worker_cpu) worker_cpu=$10
|
||||
}
|
||||
END {printf "%d,%d,%d,%d,%d,%d,%d,%d",samples,queued,running,db,allocated,node_memory,worker_memory,worker_cpu}
|
||||
' "$pressure_report")
|
||||
IFS=',' read -r samples peak_queued peak_running peak_db peak_allocated peak_node_memory peak_worker_memory peak_worker_cpu <<<"$peaks"
|
||||
jq -n \
|
||||
--arg workload "$workload" \
|
||||
--argjson slots "$slots" \
|
||||
--argjson samples "$samples" \
|
||||
--argjson peakQueued "$peak_queued" \
|
||||
--argjson peakRunning "$peak_running" \
|
||||
--argjson peakDatabaseConnections "$peak_db" \
|
||||
--argjson peakAllocatedCapacity "$peak_allocated" \
|
||||
--argjson peakNodeMemoryPercent "$peak_node_memory" \
|
||||
--argjson peakWorkerMemoryMiB "$peak_worker_memory" \
|
||||
--argjson peakWorkerCPUMillicores "$peak_worker_cpu" \
|
||||
--slurpfile load "$load_report" \
|
||||
'{
|
||||
workload:$workload,
|
||||
configuredSlots:$slots,
|
||||
samples:$samples,
|
||||
peakQueued:$peakQueued,
|
||||
peakRunning:$peakRunning,
|
||||
peakDatabaseConnections:$peakDatabaseConnections,
|
||||
peakAllocatedCapacity:$peakAllocatedCapacity,
|
||||
peakNodeMemoryPercent:$peakNodeMemoryPercent,
|
||||
peakWorkerMemoryMiB:$peakWorkerMemoryMiB,
|
||||
peakWorkerCPUMillicores:$peakWorkerCPUMillicores,
|
||||
load:$load[0]
|
||||
}' >"$summary"
|
||||
chmod 0600 "$summary"
|
||||
}
|
||||
|
||||
run_single_node_profile() {
|
||||
local workload=$1
|
||||
local profile=$2
|
||||
local slots=$3
|
||||
local requests=$4
|
||||
local prefix=$report_root/single-node-$workload-s$slots
|
||||
local load_report=$prefix-load.json
|
||||
local pressure_report=$prefix-pressure.csv
|
||||
local pressure_failure=$prefix-pressure.failure.csv
|
||||
local pressure_stop=$temporary_root/single-pressure-stop
|
||||
local sampler_pid status=0 sampler_status=0
|
||||
rm -f -- "$pressure_stop" "$pressure_failure"
|
||||
sample_single_node_pressure "$pressure_report" "$pressure_stop" "$pressure_failure" &
|
||||
sampler_pid=$!
|
||||
active_pressure_pid=$sampler_pid
|
||||
run_with_pressure_monitor "$sampler_pid" \
|
||||
run_single_node_load_profile "$profile" "$load_report" "$requests" || status=$?
|
||||
touch "$pressure_stop"
|
||||
wait "$sampler_pid" || sampler_status=$?
|
||||
active_pressure_pid=
|
||||
(( status == 0 && sampler_status == 0 )) || {
|
||||
[[ -s $pressure_failure ]] && cp "$pressure_failure" "$report_root/last-pressure-failure.csv"
|
||||
return 1
|
||||
}
|
||||
record_single_node_result "$workload" "$slots" "$load_report" "$pressure_report" \
|
||||
"$prefix-summary.json"
|
||||
}
|
||||
|
||||
finish_single_node_diagnostic() {
|
||||
local diagnostic_status=$1
|
||||
local reason=$2
|
||||
local emulator_report=$report_root/single-node-emulator-report.json
|
||||
local callback_report=$report_root/single-node-callback-report.json
|
||||
local finish_response=$temporary_root/single-node-finish.json
|
||||
if [[ $single_node_emulator_installed == true ]]; then
|
||||
cluster_ssh "$CLUSTER_NINGBO_HOST" \
|
||||
'curl -fsS --max-time 10 http://127.0.0.1:18090/report' >"$emulator_report" || true
|
||||
else
|
||||
remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-emulator -- \
|
||||
wget -qO- http://127.0.0.1:8090/report >"$emulator_report" || true
|
||||
fi
|
||||
remote_kubectl exec -n "$namespace" deployment/easyai-acceptance-callback-collector -- \
|
||||
wget -qO- http://127.0.0.1:8091/report >"$callback_report" || true
|
||||
local task_state
|
||||
task_state=$(database_query "
|
||||
SELECT count(*)||':'||count(*) FILTER (WHERE status='succeeded')||':'||count(*) FILTER (WHERE status IN ('queued','running'))
|
||||
FROM gateway_tasks WHERE acceptance_run_id='$run_id'::uuid;")
|
||||
local results_json=$temporary_root/single-node-results.json
|
||||
local -a result_files=()
|
||||
shopt -s nullglob
|
||||
result_files=("$report_root"/single-node-*-summary.json)
|
||||
shopt -u nullglob
|
||||
if (( ${#result_files[@]} > 0 )); then
|
||||
jq -s '.' "${result_files[@]}" >"$results_json"
|
||||
else
|
||||
printf '[]\n' >"$results_json"
|
||||
fi
|
||||
jq -n \
|
||||
--arg runId "$run_id" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg status "$diagnostic_status" \
|
||||
--arg reason "$reason" \
|
||||
--arg taskState "$task_state" \
|
||||
--argjson stableSlots "$single_node_stable_slots" \
|
||||
--slurpfile results "$results_json" \
|
||||
'{
|
||||
schemaVersion:"acceptance-single-node-diagnostic/v1",
|
||||
runId:$runId,
|
||||
releaseSha:$releaseSha,
|
||||
site:"ningbo",
|
||||
mode:"acceptance-emulator-only",
|
||||
realUpstreamRequests:0,
|
||||
certification:false,
|
||||
promotable:false,
|
||||
status:$status,
|
||||
reason:$reason,
|
||||
taskState:$taskState,
|
||||
stableVideoSlots:$stableSlots,
|
||||
results:$results[0]
|
||||
}' >"$report_root/single-node-summary.json"
|
||||
chmod 0600 "$report_root/single-node-summary.json"
|
||||
local finish_body
|
||||
finish_body=$(jq -cn \
|
||||
--arg reason "single-node diagnostic completed; real upstream and certification intentionally skipped" \
|
||||
--arg status "$diagnostic_status" \
|
||||
--argjson stableSlots "$single_node_stable_slots" \
|
||||
'{passed:false,failureReason:$reason,report:{diagnosticStatus:$status,site:"ningbo",realUpstreamRequests:0,certification:false,stableVideoSlots:$stableSlots}}')
|
||||
admin_request POST "/api/admin/system/acceptance/runs/$run_id/finish" "$finish_body" "$finish_response"
|
||||
failure_recorded=true
|
||||
cleanup_single_node_emulator
|
||||
remote_kubectl delete deployment easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
remote_kubectl delete service easyai-acceptance-emulator easyai-acceptance-callback-collector \
|
||||
-n "$namespace" --ignore-not-found >/dev/null || true
|
||||
}
|
||||
|
||||
run_single_node_acceptance() {
|
||||
wait_for_existing_tasks_to_drain
|
||||
single_node_stable_slots=$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_BASELINE_SLOTS
|
||||
apply_single_node_capacity "$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS"
|
||||
if ! run_single_node_profile gemini-multi-image gemini-multi-image \
|
||||
"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_SLOTS" \
|
||||
"$AI_GATEWAY_ACCEPTANCE_SINGLE_NODE_GEMINI_REQUESTS"; then
|
||||
finish_single_node_diagnostic failed 'GEMINI multi-image diagnostic failed or crossed a resource gate'
|
||||
return 1
|
||||
fi
|
||||
local slots requests
|
||||
for slots in "${single_node_video_slots[@]}"; do
|
||||
apply_single_node_capacity "$slots" || break
|
||||
requests=$((slots * 3))
|
||||
(( requests < 32 )) && requests=32
|
||||
if ! run_single_node_profile image-video video-throughput "$slots" "$requests"; then
|
||||
break
|
||||
fi
|
||||
single_node_stable_slots=$slots
|
||||
done
|
||||
apply_single_node_capacity "$single_node_stable_slots"
|
||||
finish_single_node_diagnostic completed 'single-node emulator-only capacity ladder completed'
|
||||
echo "single_node_acceptance=PASS_DIAGNOSTIC run_id=$run_id stable_video_slots=$single_node_stable_slots traffic_mode=validation certification=false report=$report_root/single-node-summary.json"
|
||||
}
|
||||
|
||||
run_capacity_round() {
|
||||
local profile=$1
|
||||
local repetition=$2
|
||||
@@ -3651,11 +4178,17 @@ verify_release_cas false
|
||||
bootstrap_acceptance_primary_identity
|
||||
ensure_acceptance_user_group
|
||||
ensure_acceptance_identity_shards
|
||||
ensure_acceptance_real_images
|
||||
if [[ $single_node_acceptance != true ]]; then
|
||||
ensure_acceptance_real_images
|
||||
fi
|
||||
select_acceptance_models
|
||||
ensure_acceptance_model_access
|
||||
deploy_protocol_emulator
|
||||
create_and_activate_run
|
||||
if [[ $single_node_acceptance == true ]]; then
|
||||
run_single_node_acceptance
|
||||
exit $?
|
||||
fi
|
||||
wait_for_existing_tasks_to_drain
|
||||
snapshot_pre_acceptance_capacity
|
||||
if ! apply_capacity_profile P24; then
|
||||
|
||||
Reference in New Issue
Block a user