fix(cluster): 等待 Worker 完全退出后执行跨节点验收

修正缩容期间 Deployment 已显示零就绪但旧 Worker Pod 尚在退出时仍可能领取验收任务的竞态。真实任务提交前现在会确认宁波 Worker Pod 数量为零,并在失败路径尝试取消未完成的测试任务。\n\n验证:node --check、bash -n、ShellCheck、git diff --check。
This commit is contained in:
2026-07-29 17:29:15 +08:00
parent a073c840ac
commit de1ce2274b
2 changed files with 14 additions and 3 deletions
+7 -2
View File
@@ -26,24 +26,29 @@ set -euo pipefail
k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null
k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=0 >/dev/null
for _ in $(seq 1 60); do
for _ in $(seq 1 180); do
pods=$(k3s kubectl get pods -n easyai \
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
ningbo_pods=$(k3s kubectl get pods -n easyai \
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo \
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
ningbo_pod_count=$(wc -w <<<"$ningbo_pods" | tr -d ' ')
if [[ $pod_count == 1 ]]; then
pod=$pods
process_role=$(k3s kubectl get pod "$pod" -n easyai \
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}')
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
-o jsonpath='{.status.containerStatuses[0].ready}')
if [[ $process_role == worker && $pod_ready == true ]]; then
if [[ $process_role == worker && $pod_ready == true && $ningbo_pod_count == 0 ]]; then
break
fi
fi
sleep 1
done
[[ ${pod_count:-0} == 1 && ${process_role:-} == worker && ${pod_ready:-} == true ]]
[[ ${ningbo_pod_count:-1} -eq 0 ]]
[[ $(k3s kubectl get deployment easyai-api-ningbo -n easyai \
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
ningbo_worker_ready=$(k3s kubectl get deployment easyai-worker-ningbo -n easyai \