fix(cluster): 等待 Worker 完全退出后执行跨节点验收
修正缩容期间 Deployment 已显示零就绪但旧 Worker Pod 尚在退出时仍可能领取验收任务的竞态。真实任务提交前现在会确认宁波 Worker Pod 数量为零,并在失败路径尝试取消未完成的测试任务。\n\n验证:node --check、bash -n、ShellCheck、git diff --check。
This commit is contained in:
@@ -129,6 +129,8 @@ assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creat
|
|||||||
assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable')
|
assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable')
|
||||||
|
|
||||||
let result
|
let result
|
||||||
|
let taskId
|
||||||
|
let taskSucceeded = false
|
||||||
try {
|
try {
|
||||||
const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt })
|
const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt })
|
||||||
const targetModels = (assignableModels.payload.items || []).filter(
|
const targetModels = (assignableModels.payload.items || []).filter(
|
||||||
@@ -185,7 +187,7 @@ try {
|
|||||||
form,
|
form,
|
||||||
})
|
})
|
||||||
assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`)
|
assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`)
|
||||||
const taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id
|
taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id
|
||||||
assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID')
|
assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID')
|
||||||
|
|
||||||
let task
|
let task
|
||||||
@@ -202,8 +204,12 @@ try {
|
|||||||
assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`)
|
assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`)
|
||||||
const resultURLs = [...new Set(collectHTTPSURLs(task.result))]
|
const resultURLs = [...new Set(collectHTTPSURLs(task.result))]
|
||||||
assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL')
|
assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL')
|
||||||
|
taskSucceeded = true
|
||||||
result = { taskId, sourceSHA256, resultURL: resultURLs[0], model }
|
result = { taskId, sourceSHA256, resultURL: resultURLs[0], model }
|
||||||
} finally {
|
} finally {
|
||||||
|
if (taskId && !taskSucceeded) {
|
||||||
|
await request(`/api/v1/tasks/${taskId}/cancel`, { method: 'POST', token: apiKey }).catch(() => {})
|
||||||
|
}
|
||||||
if (apiKeyId) {
|
if (apiKeyId) {
|
||||||
await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {})
|
await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,24 +26,29 @@ set -euo pipefail
|
|||||||
k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null
|
k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null
|
||||||
k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
|
k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s
|
||||||
k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=0 >/dev/null
|
k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=0 >/dev/null
|
||||||
for _ in $(seq 1 60); do
|
for _ in $(seq 1 180); do
|
||||||
pods=$(k3s kubectl get pods -n easyai \
|
pods=$(k3s kubectl get pods -n easyai \
|
||||||
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong \
|
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong \
|
||||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||||
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
|
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
|
||||||
|
ningbo_pods=$(k3s kubectl get pods -n easyai \
|
||||||
|
-l app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo \
|
||||||
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||||
|
ningbo_pod_count=$(wc -w <<<"$ningbo_pods" | tr -d ' ')
|
||||||
if [[ $pod_count == 1 ]]; then
|
if [[ $pod_count == 1 ]]; then
|
||||||
pod=$pods
|
pod=$pods
|
||||||
process_role=$(k3s kubectl get pod "$pod" -n easyai \
|
process_role=$(k3s kubectl get pod "$pod" -n easyai \
|
||||||
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}')
|
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}')
|
||||||
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
|
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
|
||||||
-o jsonpath='{.status.containerStatuses[0].ready}')
|
-o jsonpath='{.status.containerStatuses[0].ready}')
|
||||||
if [[ $process_role == worker && $pod_ready == true ]]; then
|
if [[ $process_role == worker && $pod_ready == true && $ningbo_pod_count == 0 ]]; then
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
[[ ${pod_count:-0} == 1 && ${process_role:-} == worker && ${pod_ready:-} == true ]]
|
[[ ${pod_count:-0} == 1 && ${process_role:-} == worker && ${pod_ready:-} == true ]]
|
||||||
|
[[ ${ningbo_pod_count:-1} -eq 0 ]]
|
||||||
[[ $(k3s kubectl get deployment easyai-api-ningbo -n easyai \
|
[[ $(k3s kubectl get deployment easyai-api-ningbo -n easyai \
|
||||||
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
|
-o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]]
|
||||||
ningbo_worker_ready=$(k3s kubectl get deployment easyai-worker-ningbo -n easyai \
|
ningbo_worker_ready=$(k3s kubectl get deployment easyai-worker-ningbo -n easyai \
|
||||||
|
|||||||
Reference in New Issue
Block a user