feat(deploy): 增加三节点 K3s 高可用迁移能力

新增 WireGuard 全互联、三 server embedded-etcd K3s、CloudNativePG 双实例、Barman OSS 备份、双 NGINX、Kubernetes Secret/RBAC 与本地旧文件按严格 24 小时清理。新增维护窗口数据迁移、digest 固定滚动发布、应用回滚、跨节点文件 E2E、节点和数据库故障演练、CNPG 恢复与 etcd 快照验收脚本；洛杉矶仅作为带 NoSchedule 污点的仲裁节点。所有生产 Secret 只在执行时从 0600 本地环境和旧生产容器导入，仓库不保存凭据；公网入口保持人工 DNS 故障切换边界。验证：bash -n、ShellCheck、kubectl kustomize、Node 语法检查、Secret 扫描、OSS put/head/delete 实测。
2026-07-28 04:37:31 +08:00
parent edf7e66941
commit 971540a2a4
38 changed files with 3386 additions and 3 deletions
@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+import { createHash, randomUUID } from 'node:crypto'
+import { deflateSync } from 'node:zlib'
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message)
+}
+
+function crc32(buffer) {
+  let value = 0xffffffff
+  for (const byte of buffer) {
+    value ^= byte
+    for (let bit = 0; bit < 8; bit += 1) {
+      value = (value >>> 1) ^ (value & 1 ? 0xedb88320 : 0)
+    }
+  }
+  return (value ^ 0xffffffff) >>> 0
+}
+
+function pngChunk(type, data) {
+  const typeBuffer = Buffer.from(type)
+  const length = Buffer.alloc(4)
+  length.writeUInt32BE(data.length)
+  const checksum = Buffer.alloc(4)
+  checksum.writeUInt32BE(crc32(Buffer.concat([typeBuffer, data])))
+  return Buffer.concat([length, typeBuffer, data, checksum])
+}
+
+function createFixturePNG(width = 256, height = 256) {
+  const header = Buffer.alloc(13)
+  header.writeUInt32BE(width, 0)
+  header.writeUInt32BE(height, 4)
+  header[8] = 8
+  header[9] = 2
+  const rows = []
+  for (let y = 0; y < height; y += 1) {
+    const row = Buffer.alloc(1 + width * 3)
+    for (let x = 0; x < width; x += 1) {
+      row[1 + x * 3] = Math.round((x / width) * 255)
+      row[2 + x * 3] = Math.round((y / height) * 255)
+      row[3 + x * 3] = 180
+    }
+    rows.push(row)
+  }
+  return Buffer.concat([
+    Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+    pngChunk('IHDR', header),
+    pngChunk('IDAT', deflateSync(Buffer.concat(rows))),
+    pngChunk('IEND', Buffer.alloc(0)),
+  ])
+}
+
+const baseURL = process.env.AI_GATEWAY_E2E_BASE_URL || 'http://10.77.0.1:18088'
+const account = Buffer.from(process.env.E2E_ACCOUNT_B64 || '', 'base64').toString('utf8')
+const password = Buffer.from(process.env.E2E_PASSWORD_B64 || '', 'base64').toString('utf8')
+const model = process.env.AI_GATEWAY_E2E_IMAGE_MODEL || 'openai:gpt-image-1'
+assert(account && password, 'E2E login credentials are unavailable')
+
+async function request(path, { method = 'GET', token, body, form } = {}) {
+  const headers = { Host: 'ai.51easyai.com' }
+  if (token) headers.Authorization = `Bearer ${token}`
+  if (body !== undefined) headers['Content-Type'] = 'application/json'
+  if (form) headers['X-Async'] = 'true'
+  const response = await fetch(`${baseURL}${path}`, {
+    method,
+    headers,
+    body: form || (body === undefined ? undefined : JSON.stringify(body)),
+    signal: AbortSignal.timeout(3_600_000),
+  })
+  const text = await response.text()
+  let payload
+  try {
+    payload = text ? JSON.parse(text) : {}
+  } catch {
+    payload = { raw: text.slice(0, 500) }
+  }
+  if (!response.ok) {
+    throw new Error(`${method} ${path} returned HTTP ${response.status}: ${JSON.stringify(payload)}`)
+  }
+  return { response, payload }
+}
+
+function collectHTTPSURLs(value, output = []) {
+  if (typeof value === 'string' && value.startsWith('https://')) output.push(value)
+  else if (Array.isArray(value)) value.forEach((item) => collectHTTPSURLs(item, output))
+  else if (value && typeof value === 'object') Object.values(value).forEach((item) => collectHTTPSURLs(item, output))
+  return output
+}
+
+const login = await request('/api/v1/auth/login', {
+  method: 'POST',
+  body: { account, password },
+})
+const jwt = login.payload.accessToken
+assert(typeof jwt === 'string' && jwt.length > 20, 'login did not return an access token')
+
+const keyName = `cluster-cross-node-${Date.now()}`
+const keyResult = await request('/api/v1/api-keys', {
+  method: 'POST',
+  token: jwt,
+  body: { name: keyName },
+})
+const apiKey = keyResult.payload.secret
+const apiKeyId = keyResult.payload.id || keyResult.payload.item?.id
+assert(typeof apiKey === 'string' && apiKey.startsWith('sk-gw-'), 'API key creation failed')
+
+let result
+try {
+  const fixture = createFixturePNG()
+  const sourceSHA256 = createHash('sha256').update(fixture).digest('hex')
+  const form = new FormData()
+  form.append('model', model)
+  form.append('prompt', `Cross-node HA acceptance ${randomUUID()}: add a thin white border`)
+  form.append('size', '1024x1024')
+  form.append('image', new Blob([fixture], { type: 'image/png' }), 'cluster-e2e.png')
+  const accepted = await request('/api/v1/images/edits', {
+    method: 'POST',
+    token: apiKey,
+    form,
+  })
+  assert(accepted.response.status === 202, `image edit returned ${accepted.response.status}, expected 202`)
+  const taskId = accepted.payload.taskId || accepted.payload.task_id || accepted.payload.task?.id
+  assert(/^[0-9a-f-]{36}$/.test(taskId || ''), 'image edit response has no task ID')
+
+  let task
+  const deadline = Date.now() + 20 * 60_000
+  while (Date.now() < deadline) {
+    const taskResult = await request(`/api/v1/tasks/${taskId}`, { token: apiKey })
+    task = taskResult.payload.task || taskResult.payload
+    if (task.status === 'succeeded') break
+    if (['failed', 'cancelled'].includes(task.status)) {
+      throw new Error(`task ${taskId} ended as ${task.status}: ${task.errorMessage || task.error || ''}`)
+    }
+    await new Promise((resolve) => setTimeout(resolve, 3000))
+  }
+  assert(task?.status === 'succeeded', `task ${taskId} did not succeed before timeout`)
+  const resultURLs = [...new Set(collectHTTPSURLs(task.result))]
+  assert(resultURLs.length > 0, 'task result contains no shared HTTPS URL')
+  result = { taskId, sourceSHA256, resultURL: resultURLs[0], model }
+} finally {
+  if (apiKeyId) {
+    await request(`/api/v1/api-keys/${apiKeyId}`, { method: 'DELETE', token: jwt }).catch(() => {})
+  }
+}
+
+console.log(JSON.stringify(result))