feat(storage): 完善对象存储配置与过期策略

补齐 OSS/S3 的 Endpoint、Region、Bucket、CDN、对象前缀和签名有效期配置，并为生成结果与请求素材自动维护分级生命周期规则。普通上传继续保持永久，私有资源按配置生成限时签名 URL，管理端连接测试覆盖生命周期、上传、读取和删除。\n\n新增可重复的真实 OSS 验收脚本，凭据仅从本地环境读取，接口响应继续保持脱敏。\n\n验证：Go 全量测试、迁移安全检查、pnpm lint、pnpm test、pnpm build、本地阿里云 OSS 真实上传下载删除验收。
2026-08-04 12:44:24 +08:00
parent fe56aa46b9
commit 6a64d3936a
16 changed files with 1274 additions and 47 deletions
@@ -0,0 +1,275 @@
+#!/usr/bin/env node
+
+import { createHmac } from 'node:crypto';
+
+const requiredEnvironment = [
+  'AI_GATEWAY_TEST_OSS_ENDPOINT',
+  'AI_GATEWAY_TEST_OSS_REGION',
+  'AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID',
+  'AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET',
+  'AI_GATEWAY_TEST_OSS_BUCKET',
+];
+
+for (const key of requiredEnvironment) {
+  if (!String(process.env[key] ?? '').trim()) {
+    throw new Error(`missing required environment: ${key}`);
+  }
+}
+
+const gatewayBaseURL = String(process.env.AI_GATEWAY_TEST_BASE_URL ?? 'http://127.0.0.1:8088').replace(/\/$/, '');
+const endpoint = new URL(process.env.AI_GATEWAY_TEST_OSS_ENDPOINT);
+const region = process.env.AI_GATEWAY_TEST_OSS_REGION.trim();
+const accessKeyID = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_ID.trim();
+const accessKeySecret = process.env.AI_GATEWAY_TEST_OSS_ACCESS_KEY_SECRET.trim();
+const bucket = process.env.AI_GATEWAY_TEST_OSS_BUCKET.trim();
+const expirationPolicy = String(process.env.AI_GATEWAY_TEST_OSS_EXPIRATION_POLICY ?? '1d').trim();
+const signedURLExpiresSeconds = Number(process.env.AI_GATEWAY_TEST_OSS_SIGNED_URL_EXPIRES_SECONDS ?? 900);
+const channelKey = String(
+  process.env.AI_GATEWAY_TEST_OSS_CHANNEL_KEY ?? `aliyun-oss-${region}-${bucket}-local`,
+).trim();
+const channelName = String(
+  process.env.AI_GATEWAY_TEST_OSS_CHANNEL_NAME ?? `Aliyun OSS ${region} ${bucket} 本地验收`,
+).trim();
+const managedLifecycleRuleIDs = [
+  'DeleteTempFiles-1d',
+  'DeleteTempFiles-1m',
+  'DeleteTempFiles-3m',
+  'DeleteTempFiles-6m',
+];
+
+function base64URL(value) {
+  return Buffer.from(value).toString('base64url');
+}
+
+function managerToken() {
+  if (!String(process.env.CONFIG_JWT_SECRET ?? '').trim()) {
+    throw new Error('missing required environment: CONFIG_JWT_SECRET');
+  }
+  const now = Math.floor(Date.now() / 1000);
+  const header = base64URL(JSON.stringify({ alg: 'HS256', typ: 'JWT' }));
+  const payload = base64URL(JSON.stringify({
+    sub: 'local-oss-acceptance-manager',
+    username: 'local-oss-acceptance-manager',
+    role: ['manager'],
+    source: 'gateway',
+    gatewayUserId: 'local-oss-acceptance-manager',
+    tokenPurpose: 'local_break_glass_manager',
+    iat: now,
+    exp: now + 15 * 60,
+  }));
+  const signature = createHmac('sha256', process.env.CONFIG_JWT_SECRET)
+    .update(`${header}.${payload}`)
+    .digest('base64url');
+  return `${header}.${payload}.${signature}`;
+}
+
+async function gatewayAuthorization() {
+  const account = String(process.env.AI_GATEWAY_ONLINE_ACCOUNT ?? '').trim();
+  const password = String(process.env.AI_GATEWAY_ONLINE_PASSWORD ?? '');
+  const gatewayHostname = new URL(gatewayBaseURL).hostname;
+  const localGateway = gatewayHostname === '127.0.0.1' || gatewayHostname === 'localhost' || gatewayHostname === '::1';
+  if (account && password && !localGateway) {
+    const response = await fetch(`${gatewayBaseURL}/api/v1/auth/login`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ account, password }),
+    });
+    const payload = await response.json().catch(() => ({}));
+    if (!response.ok || !payload.accessToken) {
+      throw new Error(`Gateway online login failed: HTTP ${response.status}`);
+    }
+    return `Bearer ${payload.accessToken}`;
+  }
+  return `Bearer ${managerToken()}`;
+}
+
+const authorization = await gatewayAuthorization();
+
+async function gatewayJSON(path, init = {}) {
+  const response = await fetch(`${gatewayBaseURL}${path}`, {
+    ...init,
+    headers: {
+      Authorization: authorization,
+      ...(init.body && !(init.body instanceof FormData) ? { 'Content-Type': 'application/json' } : {}),
+      ...init.headers,
+    },
+  });
+  const text = await response.text();
+  let payload = {};
+  if (text) {
+    try {
+      payload = JSON.parse(text);
+    } catch {
+      payload = { message: 'non-JSON response' };
+    }
+  }
+  if (!response.ok) {
+    const code = payload.code ?? payload.error?.code ?? 'unknown';
+    const message = payload.message ?? payload.error?.message ?? 'request failed';
+    throw new Error(`gateway ${init.method ?? 'GET'} ${path} failed: HTTP ${response.status} ${code} ${message}`);
+  }
+  return payload;
+}
+
+function objectURL(objectKey = '', lifecycle = false) {
+  const url = new URL(endpoint.toString());
+  url.hostname = `${bucket}.${url.hostname}`;
+  url.pathname = objectKey
+    ? `/${objectKey.split('/').map((part) => encodeURIComponent(part)).join('/')}`
+    : '/';
+  url.search = lifecycle ? '?lifecycle' : '';
+  return url;
+}
+
+async function signedOSSRequest(method, objectKey = '', lifecycle = false) {
+  const date = new Date().toUTCString();
+  const canonicalResource = lifecycle
+    ? `/${bucket}/?lifecycle`
+    : `/${bucket}/${objectKey}`;
+  const stringToSign = `${method}\n\n\n${date}\n${canonicalResource}`;
+  const signature = createHmac('sha1', accessKeySecret).update(stringToSign).digest('base64');
+  return fetch(objectURL(objectKey, lifecycle), {
+    method,
+    headers: {
+      Authorization: `OSS ${accessKeyID}:${signature}`,
+      Date: date,
+    },
+  });
+}
+
+function channelInput() {
+  return {
+    channelKey,
+    name: channelName,
+    provider: 'aliyun_oss',
+    accessKeyId: accessKeyID,
+    accessKeySecret,
+    scenes: ['upload', 'image_result', 'request_asset'],
+    config: {
+      endpoint: endpoint.toString().replace(/\/$/, ''),
+      region,
+      bucket,
+      objectKeyPrefix: 'easyai-ai-gateway/live-acceptance',
+      accessScope: 'private',
+      temporaryFileExpirePolicy: expirationPolicy,
+      signedUrlExpiresSeconds: signedURLExpiresSeconds,
+    },
+    retryPolicy: {
+      enabled: true,
+      maxRetries: 1,
+      backoffSeconds: [0.25],
+    },
+    priority: 10,
+    status: 'enabled',
+  };
+}
+
+const health = await gatewayJSON('/api/v1/healthz');
+const ready = await gatewayJSON('/api/v1/readyz');
+if (health.ok !== true || ready.ok !== true) {
+  throw new Error('gateway health or readiness check failed');
+}
+
+const listed = await gatewayJSON('/api/admin/system/file-storage/channels');
+const existing = Array.isArray(listed.items)
+  ? listed.items.find((item) => item.channelKey === channelKey)
+  : undefined;
+const channel = existing
+  ? await gatewayJSON(`/api/admin/system/file-storage/channels/${encodeURIComponent(existing.id)}`, {
+      method: 'PATCH',
+      body: JSON.stringify(channelInput()),
+    })
+  : await gatewayJSON('/api/admin/system/file-storage/channels', {
+      method: 'POST',
+      body: JSON.stringify(channelInput()),
+    });
+
+const connectionTest = await gatewayJSON(
+  `/api/admin/system/file-storage/channels/${encodeURIComponent(channel.id)}/test`,
+  { method: 'POST' },
+);
+for (const key of ['putSucceeded', 'headSucceeded', 'deleteSucceeded', 'lifecycleReady']) {
+  if (connectionTest[key] !== true) {
+    throw new Error(`object storage connection test did not prove ${key}`);
+  }
+}
+if (connectionTest.expirationPolicy !== expirationPolicy) {
+  throw new Error('object storage connection test returned an unexpected expiration policy');
+}
+
+const lifecycleResponse = await signedOSSRequest('GET', '', true);
+const lifecycleXML = await lifecycleResponse.text();
+if (!lifecycleResponse.ok) {
+  throw new Error(`OSS lifecycle read failed: HTTP ${lifecycleResponse.status}`);
+}
+const lifecycleRuleIDs = [...lifecycleXML.matchAll(/<ID>([^<]+)<\/ID>/g)].map((match) => match[1]);
+const missingLifecycleRules = managedLifecycleRuleIDs.filter((id) => !lifecycleRuleIDs.includes(id));
+if (missingLifecycleRules.length > 0) {
+  throw new Error(`OSS lifecycle rules missing: ${missingLifecycleRules.join(', ')}`);
+}
+
+const expectedContent = `easyai-oss-live-acceptance:${new Date().toISOString()}`;
+const form = new FormData();
+form.append('file', new Blob([expectedContent], { type: 'text/plain' }), 'oss-live-acceptance.txt');
+form.append('source', 'local-oss-live-acceptance');
+const upload = await gatewayJSON('/api/v1/files/upload', { method: 'POST', body: form });
+if (!upload.url || !upload.objectKey || upload.accessScope !== 'private') {
+  throw new Error('Gateway upload response did not include a private OSS object and signed URL');
+}
+if (upload.storageChannel?.channelKey !== channelKey) {
+  throw new Error(`Gateway upload did not select the expected OSS channel: ${upload.storageChannel?.channelKey ?? 'none'}`);
+}
+if (!upload.urlExpiresAt || upload.objectExpiresAt || upload.expirationPolicy) {
+  throw new Error('normal upload must have a signed URL TTL but remain exempt from temporary-file expiration');
+}
+
+const downloadResponse = await fetch(upload.url);
+const downloadedContent = await downloadResponse.text();
+if (!downloadResponse.ok || downloadedContent !== expectedContent) {
+  throw new Error(`signed URL download failed: HTTP ${downloadResponse.status}`);
+}
+
+const deleteResponse = await signedOSSRequest('DELETE', upload.objectKey);
+if (!deleteResponse.ok) {
+  throw new Error(`OSS cleanup failed: HTTP ${deleteResponse.status}`);
+}
+const afterDeleteResponse = await fetch(upload.url);
+if (afterDeleteResponse.status !== 404) {
+  throw new Error(`deleted OSS object remained readable: HTTP ${afterDeleteResponse.status}`);
+}
+
+const signedURL = new URL(upload.url);
+console.log(JSON.stringify({
+  gateway: {
+    health: health.ok,
+    ready: ready.ok,
+  },
+  channel: {
+    action: existing ? 'updated' : 'created',
+    id: channel.id,
+    channelKey: channel.channelKey,
+    provider: channel.provider,
+    status: channel.status,
+    region: channel.config?.region,
+    bucket: channel.config?.bucket,
+    expirationPolicy: channel.config?.temporaryFileExpirePolicy,
+    credentialsConfigured: Boolean(channel.credentialsPreview?.accessKeyId)
+      && Boolean(channel.credentialsPreview?.accessKeySecret),
+  },
+  connectionTest,
+  lifecycle: {
+    status: lifecycleResponse.status,
+    managedRuleIDs: managedLifecycleRuleIDs,
+  },
+  upload: {
+    status: 'success',
+    bytes: Buffer.byteLength(expectedContent),
+    accessScope: upload.accessScope,
+    signedURLHost: signedURL.host,
+    signedURLExpiresAt: upload.urlExpiresAt,
+    normalUploadIsPermanent: !upload.objectExpiresAt,
+    downloadStatus: downloadResponse.status,
+    deleteStatus: deleteResponse.status,
+    afterDeleteStatus: afterDeleteResponse.status,
+  },
+}, null, 2));