fix(deploy): 强化跨节点真实任务验收门禁
修正宁波旧 Worker 在滚动退场期间仍可能领取任务的竞态,提交任务前要求仅保留唯一且明确禁用 Worker 的新 Pod。\n\n为临时验收 Key 分配 gpt-image-2 所需的最小平台访问资源,并让 multipart 参数对齐线上 2K 图像编辑请求。已通过 bash -n、ShellCheck、node --check,并完成真实跨节点任务验收。
This commit is contained in:
@@ -58,6 +58,19 @@ const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1'
|
|||||||
const imageSize = process.env.AI_GATEWAY_E2E_IMAGE_SIZE || '2048x2048'
|
const imageSize = process.env.AI_GATEWAY_E2E_IMAGE_SIZE || '2048x2048'
|
||||||
assert(account && password, 'E2E login credentials are unavailable')
|
assert(account && password, 'E2E login credentials are unavailable')
|
||||||
assert(/^\d+x\d+$/.test(imageSize), 'E2E image size is invalid')
|
assert(/^\d+x\d+$/.test(imageSize), 'E2E image size is invalid')
|
||||||
|
const [imageWidth, imageHeight] = imageSize.split('x').map(Number)
|
||||||
|
const imageResolution =
|
||||||
|
process.env.AI_GATEWAY_E2E_IMAGE_RESOLUTION ||
|
||||||
|
(imageWidth === imageHeight && imageWidth % 1024 === 0 ? `${imageWidth / 1024}K` : '')
|
||||||
|
const imageAspectRatio =
|
||||||
|
process.env.AI_GATEWAY_E2E_IMAGE_ASPECT_RATIO ||
|
||||||
|
`${imageWidth / greatestCommonDivisor(imageWidth, imageHeight)}:${imageHeight / greatestCommonDivisor(imageWidth, imageHeight)}`
|
||||||
|
assert(imageResolution, 'E2E image resolution is unavailable')
|
||||||
|
|
||||||
|
function greatestCommonDivisor(left, right) {
|
||||||
|
while (right !== 0) [left, right] = [right, left % right]
|
||||||
|
return left
|
||||||
|
}
|
||||||
|
|
||||||
async function request(path, { method = 'GET', token, body, form } = {}) {
|
async function request(path, { method = 'GET', token, body, form } = {}) {
|
||||||
const headers = { Host: 'ai.51easyai.com' }
|
const headers = { Host: 'ai.51easyai.com' }
|
||||||
@@ -117,13 +130,55 @@ assert(typeof apiKeyId === 'string' && apiKeyId.length > 0, 'API key ID is unava
|
|||||||
|
|
||||||
let result
|
let result
|
||||||
try {
|
try {
|
||||||
|
const assignableModels = await request('/api/v1/api-keys/assignable-models', { token: jwt })
|
||||||
|
const targetModels = (assignableModels.payload.items || []).filter(
|
||||||
|
(item) =>
|
||||||
|
[item?.modelName, item?.modelAlias].includes(model) &&
|
||||||
|
Array.isArray(item?.modelType) &&
|
||||||
|
item.modelType.includes('image_edit'),
|
||||||
|
)
|
||||||
|
assert(targetModels.length > 0, `no assignable image_edit platform model found for ${model}`)
|
||||||
|
const resourceMap = new Map()
|
||||||
|
for (const item of targetModels) {
|
||||||
|
for (const [resourceType, resourceId] of [
|
||||||
|
['platform', item.platformId],
|
||||||
|
['platform_model', item.id],
|
||||||
|
['base_model', item.baseModelId],
|
||||||
|
]) {
|
||||||
|
if (typeof resourceId === 'string' && resourceId) {
|
||||||
|
resourceMap.set(`${resourceType}:${resourceId}`, {
|
||||||
|
resourceType,
|
||||||
|
resourceId,
|
||||||
|
priority: 100,
|
||||||
|
minPermissionLevel: 0,
|
||||||
|
status: 'active',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await request('/api/v1/api-keys/access-rules/batch', {
|
||||||
|
method: 'POST',
|
||||||
|
token: jwt,
|
||||||
|
body: {
|
||||||
|
subjectType: 'api_key',
|
||||||
|
subjectId: apiKeyId,
|
||||||
|
effect: 'allow',
|
||||||
|
upsertResources: [...resourceMap.values()],
|
||||||
|
deleteResources: [],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
const fixture = createFixturePNG()
|
const fixture = createFixturePNG()
|
||||||
const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
|
const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
|
||||||
const form = new FormData()
|
const form = new FormData()
|
||||||
form.append('model', model)
|
form.append('model', model)
|
||||||
form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
|
form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
|
||||||
form.append('size', imageSize)
|
form.append('size', imageSize)
|
||||||
form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
|
form.append('width', String(imageWidth))
|
||||||
|
form.append('height', String(imageHeight))
|
||||||
|
form.append('resolution', imageResolution)
|
||||||
|
form.append('aspect_ratio', imageAspectRatio)
|
||||||
|
form.append('images', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
|
||||||
const accepted = await request('/api/v1/images/edits', {
|
const accepted = await request('/api/v1/images/edits', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
token: apiKey,
|
token: apiKey,
|
||||||
|
|||||||
@@ -31,6 +31,25 @@ k3s kubectl rollout status deployment/easyai-api-hongkong -n easyai --timeout=30
|
|||||||
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
|
k3s kubectl set env deployment/easyai-api-ningbo -n easyai \
|
||||||
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
|
AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED=false >/dev/null
|
||||||
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
k3s kubectl rollout status deployment/easyai-api-ningbo -n easyai --timeout=300s
|
||||||
|
for _ in $(seq 1 60); do
|
||||||
|
pods=$(k3s kubectl get pods -n easyai \
|
||||||
|
-l app.kubernetes.io/name=easyai-api,easyai.io/site=ningbo \
|
||||||
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
||||||
|
pod_count=$(wc -w <<<"$pods" | tr -d ' ')
|
||||||
|
if [[ $pod_count == 1 ]]; then
|
||||||
|
pod=$pods
|
||||||
|
worker_enabled=$(k3s kubectl get pod "$pod" -n easyai \
|
||||||
|
-o jsonpath='{.spec.containers[0].env[?(@.name=="AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED")].value}')
|
||||||
|
pod_ready=$(k3s kubectl get pod "$pod" -n easyai \
|
||||||
|
-o jsonpath='{.status.containerStatuses[0].ready}')
|
||||||
|
if [[ $worker_enabled == false && $pod_ready == true ]]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
[[ ${pod_count:-0} == 1 && ${worker_enabled:-} == false && ${pod_ready:-} == true ]]
|
||||||
|
k3s kubectl logs "$pod" -n easyai | grep -q 'asynchronous queue worker disabled for this process'
|
||||||
curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
|
curl -fsS http://10.77.0.1:31088/api/v1/readyz | grep -q '"ok":true'
|
||||||
curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
|
curl -fsS http://10.77.0.2:31089/api/v1/readyz | grep -q '"ok":true'
|
||||||
REMOTE
|
REMOTE
|
||||||
|
|||||||
Reference in New Issue
Block a user