diff --git a/scripts/cluster/cross-node-file-e2e.mjs b/scripts/cluster/cross-node-file-e2e.mjs index 19930c7..4e9093f 100755 --- a/scripts/cluster/cross-node-file-e2e.mjs +++ b/scripts/cluster/cross-node-file-e2e.mjs @@ -129,6 +129,8 @@ assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creat assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unavailable') let result +let taskId +let taskSucceeded = false try { const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt }) const targetModels = (assignableModels.payload.items || []).filter( @@ -185,7 +187,7 @@ try { form, }) assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`) - const taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id + taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID') let task @@ -202,8 +204,12 @@ try { assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`) const resultURLs = [...new Set(collectHTTPSURLs(task.result))] assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL') + taskSucceeded = true result = { taskId, sourceSHA256, resultURL: resultURLs[0], model } } finally { + if (taskId && !taskSucceeded) { + await request(`/api/v1/tasks/${taskId}/cancel`, { method: 'POST', token: apiKey }).catch(() => {}) + } if (apiKeyId) { await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {}) } diff --git a/scripts/cluster/run-cross-node-file-e2e.sh b/scripts/cluster/run-cross-node-file-e2e.sh index 220932f..8c9ed90 100755 --- a/scripts/cluster/run-cross-node-file-e2e.sh +++ b/scripts/cluster/run-cross-node-file-e2e.sh @@ -26,24 +26,29 @@ set -euo pipefail k3s kubectl scale deployment/easyai-worker-hongkong -n easyai --replicas=1 >/dev/null k3s kubectl rollout status deployment/easyai-worker-hongkong -n easyai --timeout=300s k3s kubectl scale deployment/easyai-worker-ningbo -n easyai --replicas=0 >/dev/null -for _ in $(seq 1 60); do +for _ in $(seq 1 180); do pods=$(k3s kubectl get pods -n easyai \ -l app.kubernetes.io/name=easyai-worker,easyai.io/site=hongkong \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') pod_count=$(wc -w <<<"$pods" | tr -d ' ') + ningbo_pods=$(k3s kubectl get pods -n easyai \ + -l app.kubernetes.io/name=easyai-worker,easyai.io/site=ningbo \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + ningbo_pod_count=$(wc -w <<<"$ningbo_pods" | tr -d ' ') if [[ $pod_count == 1 ]]; then pod=$pods process_role=$(k3s kubectl get pod "$pod" -n easyai \ -o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') pod_ready=$(k3s kubectl get pod "$pod" -n easyai \ -o jsonpath='{.status.containerStatuses[0].ready}') - if [[ $process_role == worker && $pod_ready == true ]]; then + if [[ $process_role == worker && $pod_ready == true && $ningbo_pod_count == 0 ]]; then break fi fi sleep 1 done [[ ${pod_count:-0} == 1 && ${process_role:-} == worker && ${pod_ready:-} == true ]] +[[ ${ningbo_pod_count:-1} -eq 0 ]] [[ $(k3s kubectl get deployment easyai-api-ningbo -n easyai \ -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="AI_GATEWAY_PROCESS_ROLE")].value}') == api ]] ningbo_worker_ready=$(k3s kubectl get deployment easyai-worker-ningbo -n easyai \