Merge remote-tracking branch 'origin/main' into codex/production-cicd
# Conflicts: # pnpm-lock.yaml
This commit is contained in:
Executable
+452
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const baseURL = (process.env.GATEWAY_BASE_URL || 'http://127.0.0.1:8088').replace(/\/+$/, '');
|
||||
const model = process.env.DEYUN_VIDEO_MODEL || 'deyun-seedance-2.0-canary';
|
||||
const expectedPlatform = process.env.DEYUN_VIDEO_PLATFORM || '漫路(火山兼容)';
|
||||
const expectedProviderModel = process.env.DEYUN_VIDEO_PROVIDER_MODEL || 'doubao-seedance-2-0';
|
||||
const pollIntervalMs = positiveInteger(process.env.DEYUN_VIDEO_POLL_INTERVAL_MS, 5000);
|
||||
const taskTimeoutMs = positiveInteger(process.env.DEYUN_VIDEO_TASK_TIMEOUT_MS, 20 * 60 * 1000);
|
||||
const outputPath =
|
||||
process.env.DEYUN_VIDEO_E2E_OUTPUT ||
|
||||
`artifacts/deyun-video-e2e-${new Date().toISOString().replaceAll(/[:.]/g, '-')}.json`;
|
||||
const database = {
|
||||
container: process.env.GATEWAY_E2E_DB_CONTAINER || 'postgres',
|
||||
name: process.env.GATEWAY_E2E_DB_NAME || 'easyai_ai_gateway',
|
||||
user: process.env.GATEWAY_E2E_DB_USER || 'easyai',
|
||||
};
|
||||
|
||||
const validationCases = [
|
||||
{
|
||||
name: '480p-4s-audio-off',
|
||||
request: {
|
||||
model,
|
||||
prompt: 'A glossy red cube rotates slowly on a clean white studio table, locked camera, soft daylight.',
|
||||
resolution: '480p',
|
||||
ratio: '16:9',
|
||||
duration: 4,
|
||||
generate_audio: false,
|
||||
seed: 41001,
|
||||
watermark: false,
|
||||
runMode: 'real',
|
||||
},
|
||||
expected: {
|
||||
height: 480,
|
||||
duration: 4,
|
||||
audio: false,
|
||||
ratio: 16 / 9,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: '720p-6s-audio-on',
|
||||
request: {
|
||||
model,
|
||||
prompt: 'A calm ocean wave rolls toward a sandy beach at sunrise, locked camera, natural ambient sound.',
|
||||
resolution: '720p',
|
||||
ratio: '16:9',
|
||||
duration: 6,
|
||||
generate_audio: true,
|
||||
seed: 42002,
|
||||
watermark: false,
|
||||
runMode: 'real',
|
||||
},
|
||||
expected: {
|
||||
height: 720,
|
||||
duration: 6,
|
||||
audio: true,
|
||||
ratio: 16 / 9,
|
||||
},
|
||||
},
|
||||
];
|
||||
const requestedCases = new Set(
|
||||
String(process.env.DEYUN_VIDEO_CASES || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
if (requestedCases.size > 0) {
|
||||
const selectedCases = validationCases.filter((validationCase) => requestedCases.has(validationCase.name));
|
||||
if (selectedCases.length !== requestedCases.size) {
|
||||
const knownCases = validationCases.map((validationCase) => validationCase.name).join(', ');
|
||||
throw new Error(`Unknown DEYUN_VIDEO_CASES value; known cases: ${knownCases}`);
|
||||
}
|
||||
validationCases.splice(0, validationCases.length, ...selectedCases);
|
||||
}
|
||||
const submittedTaskIds = [];
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
|
||||
function positiveInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value || ''), 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function nearlyEqual(left, right, tolerance = 1e-6) {
|
||||
return Math.abs(Number(left) - Number(right)) <= tolerance;
|
||||
}
|
||||
|
||||
function runPSQL(query) {
|
||||
return execFileSync(
|
||||
'docker',
|
||||
['exec', database.container, 'psql', '-U', database.user, '-d', database.name, '-At', '-F', '\t', '-c', query],
|
||||
{ encoding: 'utf8' },
|
||||
).trim();
|
||||
}
|
||||
|
||||
function resolveGatewayAPIKey() {
|
||||
if (process.env.GATEWAY_E2E_API_KEY) {
|
||||
assert(process.env.GATEWAY_E2E_API_KEY_ID, 'Set GATEWAY_E2E_API_KEY_ID when GATEWAY_E2E_API_KEY is provided');
|
||||
const metadata = apiKeyMetadata(process.env.GATEWAY_E2E_API_KEY_ID);
|
||||
return {
|
||||
...metadata,
|
||||
secret: process.env.GATEWAY_E2E_API_KEY,
|
||||
source: 'environment',
|
||||
};
|
||||
}
|
||||
const query = `
|
||||
SELECT key.id::text,
|
||||
key.key_secret,
|
||||
key.gateway_user_id::text,
|
||||
wallet.balance::float8,
|
||||
wallet.frozen_balance::float8
|
||||
FROM gateway_api_keys key
|
||||
JOIN gateway_wallet_accounts wallet
|
||||
ON wallet.gateway_user_id = key.gateway_user_id
|
||||
AND wallet.currency = 'resource'
|
||||
AND wallet.status = 'active'
|
||||
WHERE key.deleted_at IS NULL
|
||||
AND key.status = 'active'
|
||||
AND COALESCE(key.key_secret, '') <> ''
|
||||
AND (key.expires_at IS NULL OR key.expires_at > now())
|
||||
AND (key.scopes ? 'video' OR key.scopes ? '*' OR key.scopes ? 'all')
|
||||
AND (wallet.balance - wallet.frozen_balance) > 0
|
||||
ORDER BY (wallet.balance - wallet.frozen_balance) DESC, key.created_at DESC
|
||||
LIMIT 1`;
|
||||
const fields = runPSQL(query).split('\t');
|
||||
assert(fields.length === 5 && fields[0] && fields[1], 'No recoverable video-scoped Gateway API Key with positive balance was found');
|
||||
return {
|
||||
keyId: fields[0],
|
||||
secret: fields[1],
|
||||
userId: fields[2],
|
||||
balance: Number(fields[3]),
|
||||
frozenBalance: Number(fields[4]),
|
||||
source: 'database',
|
||||
};
|
||||
}
|
||||
|
||||
function apiKeyMetadata(keyId) {
|
||||
const escapedKeyId = String(keyId).replaceAll("'", "''");
|
||||
const output = runPSQL(`
|
||||
SELECT key.id::text,
|
||||
key.gateway_user_id::text,
|
||||
wallet.balance::float8,
|
||||
wallet.frozen_balance::float8
|
||||
FROM gateway_api_keys key
|
||||
JOIN gateway_wallet_accounts wallet
|
||||
ON wallet.gateway_user_id = key.gateway_user_id
|
||||
AND wallet.currency = 'resource'
|
||||
WHERE key.id = '${escapedKeyId}'::uuid
|
||||
LIMIT 1`);
|
||||
const fields = output.split('\t');
|
||||
assert(fields.length === 4 && fields[0], `Gateway API Key ${keyId} was not found`);
|
||||
return {
|
||||
keyId: fields[0],
|
||||
userId: fields[1],
|
||||
balance: Number(fields[2]),
|
||||
frozenBalance: Number(fields[3]),
|
||||
};
|
||||
}
|
||||
|
||||
function walletState(userId) {
|
||||
const escapedUserId = String(userId).replaceAll("'", "''");
|
||||
const fields = runPSQL(`
|
||||
SELECT balance::float8, frozen_balance::float8
|
||||
FROM gateway_wallet_accounts
|
||||
WHERE gateway_user_id = '${escapedUserId}'::uuid
|
||||
AND currency = 'resource'
|
||||
LIMIT 1`).split('\t');
|
||||
assert(fields.length === 2, `Wallet for Gateway user ${userId} was not found`);
|
||||
return {
|
||||
balance: Number(fields[0]),
|
||||
frozenBalance: Number(fields[1]),
|
||||
};
|
||||
}
|
||||
|
||||
async function request(path, init = {}) {
|
||||
const response = await fetch(`${baseURL}${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
Authorization: `Bearer ${authentication.secret}`,
|
||||
...(init.body ? { 'Content-Type': 'application/json' } : {}),
|
||||
...(init.headers || {}),
|
||||
},
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`${init.method || 'GET'} ${path} failed ${response.status}: ${text}`);
|
||||
}
|
||||
return text ? JSON.parse(text) : {};
|
||||
}
|
||||
|
||||
async function createVideoTask(body) {
|
||||
const accepted = await request('/api/v1/videos/generations', {
|
||||
method: 'POST',
|
||||
headers: { 'X-Async': 'true' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const taskId = accepted.taskId || accepted.task?.id;
|
||||
assert(taskId, `Video task acceptance is missing taskId: ${JSON.stringify(accepted)}`);
|
||||
submittedTaskIds.push(taskId);
|
||||
return taskId;
|
||||
}
|
||||
|
||||
async function pollTask(taskId) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < taskTimeoutMs) {
|
||||
const task = await request(`/api/v1/tasks/${taskId}`);
|
||||
if (task.status === 'succeeded') return task;
|
||||
if (task.status === 'failed' || task.status === 'cancelled') {
|
||||
throw new Error(`Task ${taskId} ended as ${task.status}: ${task.errorMessage || task.error || JSON.stringify(task.result)}`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
||||
}
|
||||
throw new Error(`Timed out after ${taskTimeoutMs}ms waiting for task ${taskId}`);
|
||||
}
|
||||
|
||||
async function taskEvents(taskId) {
|
||||
const response = await fetch(`${baseURL}/api/v1/tasks/${taskId}/events`, {
|
||||
headers: { Authorization: `Bearer ${authentication.secret}` },
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) throw new Error(`GET task events failed ${response.status}: ${text}`);
|
||||
return text
|
||||
.split(/\r?\n\r?\n/)
|
||||
.flatMap((block) => {
|
||||
const data = block
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.startsWith('data:'))
|
||||
.map((line) => line.slice(5).trim())
|
||||
.join('\n');
|
||||
return data ? [JSON.parse(data)] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function taskPreprocessingLogs(taskId) {
|
||||
return request(`/api/v1/tasks/${taskId}/param-preprocessing`);
|
||||
}
|
||||
|
||||
function videoURLFromTask(task) {
|
||||
const data = Array.isArray(task.result?.data) ? task.result.data : [];
|
||||
return data.find((item) => item?.type === 'video')?.url || data.find((item) => item?.url)?.url || '';
|
||||
}
|
||||
|
||||
async function downloadVideo(url, filePath) {
|
||||
const response = await fetch(url);
|
||||
assert(response.ok, `Download ${url} failed ${response.status}`);
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
assert(bytes.length > 0, `Downloaded video ${url} is empty`);
|
||||
await writeFile(filePath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
function probeVideo(filePath) {
|
||||
const output = execFileSync(
|
||||
'ffprobe',
|
||||
['-v', 'error', '-show_streams', '-show_format', '-of', 'json', filePath],
|
||||
{ encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
function validateRequestSnapshot(task, expectedRequest) {
|
||||
const snapshot = task.request || {};
|
||||
for (const key of ['model', 'resolution', 'ratio', 'duration', 'generate_audio', 'seed']) {
|
||||
assert(
|
||||
snapshot[key] === expectedRequest[key],
|
||||
`Task ${task.id} request.${key}=${JSON.stringify(snapshot[key])}, expected ${JSON.stringify(expectedRequest[key])}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validateTaskAudit(task, events) {
|
||||
const attempts = Array.isArray(task.attempts) ? task.attempts : [];
|
||||
assert(attempts.length === 1, `Task ${task.id} has ${attempts.length} attempts, expected exactly one`);
|
||||
const attempt = attempts[0];
|
||||
assert(attempt.platformName === expectedPlatform, `Task ${task.id} used platform ${attempt.platformName}, expected ${expectedPlatform}`);
|
||||
assert(attempt.provider === 'volces', `Task ${task.id} used provider ${attempt.provider}, expected volces`);
|
||||
assert(
|
||||
attempt.providerModelName === expectedProviderModel,
|
||||
`Task ${task.id} used provider model ${attempt.providerModelName}, expected ${expectedProviderModel}`,
|
||||
);
|
||||
assert(task.remoteTaskId, `Task ${task.id} is missing remoteTaskId`);
|
||||
assert(attempt.requestId, `Task ${task.id} attempt is missing requestId`);
|
||||
const startedEvents = events.filter((event) => event.eventType === 'task.attempt.started');
|
||||
assert(startedEvents.length === 1, `Task ${task.id} emitted ${startedEvents.length} task.attempt.started events`);
|
||||
assert(Number(task.finalChargeAmount) > 0, `Task ${task.id} finalChargeAmount must be positive`);
|
||||
return attempt;
|
||||
}
|
||||
|
||||
function validateMedia(probe, expected, taskId) {
|
||||
const streams = Array.isArray(probe.streams) ? probe.streams : [];
|
||||
const video = streams.find((stream) => stream.codec_type === 'video');
|
||||
const audioStreams = streams.filter((stream) => stream.codec_type === 'audio');
|
||||
assert(video, `Task ${taskId} output has no video stream`);
|
||||
const width = Number(video.width);
|
||||
const height = Number(video.height);
|
||||
const duration = Number(video.duration || probe.format?.duration);
|
||||
assert(height === expected.height, `Task ${taskId} output height=${height}, expected ${expected.height}`);
|
||||
const ratioError = Math.abs(width / height - expected.ratio) / expected.ratio;
|
||||
assert(ratioError <= 0.02, `Task ${taskId} aspect ratio ${width}:${height} differs from 16:9 by ${(ratioError * 100).toFixed(2)}%`);
|
||||
assert(Number.isFinite(duration), `Task ${taskId} output duration is unavailable`);
|
||||
assert(Math.abs(duration - expected.duration) <= 1, `Task ${taskId} duration=${duration}s, expected ${expected.duration}s ±1s`);
|
||||
if (expected.audio) {
|
||||
assert(audioStreams.length > 0, `Task ${taskId} requested audio but output has no audio stream`);
|
||||
const audioDuration = Number(audioStreams[0].duration || probe.format?.duration);
|
||||
if (Number.isFinite(audioDuration)) {
|
||||
assert(audioDuration >= duration - 1, `Task ${taskId} audio duration=${audioDuration}s is too short for video duration=${duration}s`);
|
||||
}
|
||||
} else {
|
||||
assert(audioStreams.length === 0, `Task ${taskId} disabled audio but output has ${audioStreams.length} audio stream(s)`);
|
||||
}
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
duration,
|
||||
ratio: width / height,
|
||||
videoCodec: video.codec_name || null,
|
||||
audioStreams: audioStreams.map((stream) => ({
|
||||
codec: stream.codec_name || null,
|
||||
channels: Number(stream.channels || 0),
|
||||
sampleRate: Number(stream.sample_rate || 0),
|
||||
duration: Number(stream.duration || probe.format?.duration || 0),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const authentication = resolveGatewayAPIKey();
|
||||
|
||||
function sanitizedFailureMessage(error) {
|
||||
return String(error instanceof Error ? error.message : error)
|
||||
.replace(/Request id:\s*[A-Za-z0-9_-]+/gi, 'Request id: [redacted]')
|
||||
.replace(/\b\d{8,}\b/g, '[redacted-id]');
|
||||
}
|
||||
|
||||
async function writeFailureReport(error) {
|
||||
const report = {
|
||||
ok: false,
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseURL,
|
||||
model,
|
||||
expectedPlatform,
|
||||
expectedProviderModel,
|
||||
authentication: {
|
||||
type: 'gateway_api_key',
|
||||
source: authentication.source,
|
||||
keyId: authentication.keyId,
|
||||
},
|
||||
submittedTaskIds,
|
||||
error: sanitizedFailureMessage(error),
|
||||
};
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
return report;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const tempDirectory = await mkdtemp(join(tmpdir(), 'deyun-video-e2e-'));
|
||||
const walletBefore = walletState(authentication.userId);
|
||||
const results = [];
|
||||
try {
|
||||
for (const validationCase of validationCases) {
|
||||
const taskId = await createVideoTask(validationCase.request);
|
||||
const task = await pollTask(taskId);
|
||||
const [events, preprocessing] = await Promise.all([
|
||||
taskEvents(taskId),
|
||||
taskPreprocessingLogs(taskId),
|
||||
]);
|
||||
validateRequestSnapshot(task, validationCase.request);
|
||||
const attempt = validateTaskAudit(task, events);
|
||||
assert(
|
||||
Array.isArray(preprocessing.items) && preprocessing.items.length > 0,
|
||||
`Task ${taskId} has no parameter preprocessing audit record`,
|
||||
);
|
||||
const videoURL = videoURLFromTask(task);
|
||||
assert(videoURL, `Task ${taskId} result is missing a video URL`);
|
||||
const videoPath = join(tempDirectory, `${validationCase.name}.mp4`);
|
||||
const byteSize = await downloadVideo(videoURL, videoPath);
|
||||
const probe = probeVideo(videoPath);
|
||||
const observed = validateMedia(probe, validationCase.expected, taskId);
|
||||
results.push({
|
||||
name: validationCase.name,
|
||||
taskId,
|
||||
remoteTaskId: task.remoteTaskId,
|
||||
requestId: attempt.requestId,
|
||||
platform: attempt.platformName,
|
||||
providerModelName: attempt.providerModelName,
|
||||
requested: {
|
||||
resolution: validationCase.request.resolution,
|
||||
ratio: validationCase.request.ratio,
|
||||
duration: validationCase.request.duration,
|
||||
generateAudio: validationCase.request.generate_audio,
|
||||
seed: validationCase.request.seed,
|
||||
},
|
||||
observed,
|
||||
byteSize,
|
||||
finalChargeAmount: Number(task.finalChargeAmount),
|
||||
billingSummary: task.billingSummary || {},
|
||||
eventTypes: events.map((event) => event.eventType),
|
||||
preprocessingChangeCount: preprocessing.items.reduce((sum, item) => sum + Number(item.changeCount || 0), 0),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await rm(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const walletAfter = walletState(authentication.userId);
|
||||
const totalCharge = results.reduce((sum, result) => sum + result.finalChargeAmount, 0);
|
||||
const walletDebit = walletBefore.balance - walletAfter.balance;
|
||||
assert(nearlyEqual(walletDebit, totalCharge, 1e-6), `Wallet debit=${walletDebit}, expected total charge=${totalCharge}`);
|
||||
assert(
|
||||
nearlyEqual(walletAfter.frozenBalance, walletBefore.frozenBalance, 1e-6),
|
||||
`Wallet frozen balance changed from ${walletBefore.frozenBalance} to ${walletAfter.frozenBalance}`,
|
||||
);
|
||||
|
||||
const report = {
|
||||
ok: true,
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseURL,
|
||||
model,
|
||||
expectedPlatform,
|
||||
expectedProviderModel,
|
||||
authentication: {
|
||||
type: 'gateway_api_key',
|
||||
source: authentication.source,
|
||||
keyId: authentication.keyId,
|
||||
},
|
||||
wallet: {
|
||||
balanceBefore: walletBefore.balance,
|
||||
balanceAfter: walletAfter.balance,
|
||||
frozenBalanceBefore: walletBefore.frozenBalance,
|
||||
frozenBalanceAfter: walletAfter.frozenBalance,
|
||||
debit: walletDebit,
|
||||
totalCharge,
|
||||
},
|
||||
results,
|
||||
};
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
console.log(JSON.stringify({ ...report, outputPath }, null, 2));
|
||||
}
|
||||
|
||||
main().catch(async (error) => {
|
||||
const report = await writeFailureReport(error);
|
||||
console.error(report.error);
|
||||
console.error(`Failure report: ${outputPath}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
|
||||
const baseURL = (process.env.GATEWAY_BASE_URL || 'http://127.0.0.1:8088').replace(/\/+$/, '');
|
||||
const authentication = resolveGatewayAPIKey();
|
||||
const timeoutMs = Number(process.env.GATEWAY_E2E_TIMEOUT_MS || 180_000);
|
||||
|
||||
const scenarios = [
|
||||
{
|
||||
name: '火山 128K Chat 未传上限',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', value: 43690, injected: true },
|
||||
},
|
||||
{
|
||||
name: '火山 32K Chat 未传上限',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_32K_MODEL || 'volces-openai:doubao-seed-1-8-251228',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', value: 10922, injected: true },
|
||||
},
|
||||
{
|
||||
name: '火山 Chat max_tokens=null',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody({ max_tokens: null }),
|
||||
expected: { parameter: 'max_tokens', value: 43690, injected: true },
|
||||
},
|
||||
{
|
||||
name: '火山 Chat 显式 max_tokens',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody({ max_tokens: 64 }),
|
||||
expected: { parameter: 'max_tokens', value: 64, injected: false },
|
||||
},
|
||||
{
|
||||
name: '火山 Chat 显式 max_completion_tokens',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody({ max_completion_tokens: 64 }),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false, preserved: { max_completion_tokens: 64 } },
|
||||
},
|
||||
{
|
||||
name: '火山 Responses 未传上限',
|
||||
platform: '火山引擎',
|
||||
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
||||
path: '/v1/responses',
|
||||
body: responsesBody(),
|
||||
expected: { parameter: 'max_output_tokens', value: 43690, injected: true },
|
||||
},
|
||||
{
|
||||
name: '阿里百炼 Qwen Chat 未传上限',
|
||||
platform: '阿里云百炼',
|
||||
model: process.env.GATEWAY_E2E_QWEN_MODEL || 'aliyun-bailian-openai:qwen3.7-plus',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: '阿里百炼 Qwen Responses 未传上限',
|
||||
platform: '阿里云百炼',
|
||||
model: process.env.GATEWAY_E2E_QWEN_MODEL || 'aliyun-bailian-openai:qwen3.7-plus',
|
||||
path: '/v1/responses',
|
||||
body: responsesBody(),
|
||||
expected: { parameter: 'max_output_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: 'DeepSeek 官网 Chat 未传上限',
|
||||
platform: 'DeepSeek 官网',
|
||||
model: process.env.GATEWAY_E2E_DEEPSEEK_MODEL || 'deepseek-openai:deepseek-v4-pro',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: 'DeepSeek 官网 Responses 回退未传上限',
|
||||
platform: 'DeepSeek 官网',
|
||||
model: process.env.GATEWAY_E2E_DEEPSEEK_MODEL || 'deepseek-openai:deepseek-v4-pro',
|
||||
path: '/v1/responses',
|
||||
body: responsesBody(),
|
||||
expected: { parameter: 'max_output_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: '智谱 GLM-5.2 Chat 未传上限',
|
||||
platform: '智谱AI',
|
||||
model: process.env.GATEWAY_E2E_ZHIPU_MODEL || 'zhipu-openai:glm-5.2',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: 'OpenAI GPT-4o Chat 未传上限',
|
||||
platform: 'OpenAI',
|
||||
model: process.env.GATEWAY_E2E_OPENAI_MODEL || 'openai:gpt-4o',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
||||
},
|
||||
{
|
||||
name: 'Google Gemini Chat 未传上限',
|
||||
platform: 'Google Gemini',
|
||||
model: process.env.GATEWAY_E2E_GEMINI_MODEL || 'gemini:gemini-2.5-pro',
|
||||
path: '/v1/chat/completions',
|
||||
body: chatBody(),
|
||||
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
||||
},
|
||||
];
|
||||
|
||||
function chatBody(extra = {}) {
|
||||
return {
|
||||
messages: [{ role: 'user', content: '这是输出上限链路验证。只回复 OK。' }],
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function responsesBody(extra = {}) {
|
||||
return { input: '这是输出上限链路验证。只回复 OK。', ...extra };
|
||||
}
|
||||
|
||||
function resolveGatewayAPIKey() {
|
||||
if (process.env.GATEWAY_E2E_API_KEY) {
|
||||
return { value: process.env.GATEWAY_E2E_API_KEY, source: 'environment', keyId: null };
|
||||
}
|
||||
const container = process.env.GATEWAY_E2E_DB_CONTAINER || 'postgres';
|
||||
const database = process.env.GATEWAY_E2E_DB_NAME || 'easyai_ai_gateway';
|
||||
const user = process.env.GATEWAY_E2E_DB_USER || 'easyai';
|
||||
const query = `SELECT key.id::text || E'\\t' || key.key_secret FROM gateway_api_keys key JOIN gateway_wallet_accounts wallet ON wallet.gateway_user_id = key.gateway_user_id AND wallet.status = 'active' WHERE key.deleted_at IS NULL AND key.status = 'active' AND key.key_secret IS NOT NULL AND key.key_secret <> '' AND key.scopes ? 'chat' AND (key.expires_at IS NULL OR key.expires_at > now()) AND (wallet.balance - wallet.frozen_balance) > 0 ORDER BY (wallet.balance - wallet.frozen_balance) DESC, key.created_at DESC LIMIT 1`;
|
||||
let output = '';
|
||||
try {
|
||||
output = execFileSync('docker', ['exec', container, 'psql', '-U', user, '-d', database, '-At', '-c', query], { encoding: 'utf8' }).trim();
|
||||
} catch {
|
||||
throw new Error('Unable to load an active chat-scoped Gateway API Key; set GATEWAY_E2E_API_KEY explicitly');
|
||||
}
|
||||
const separator = output.indexOf('\t');
|
||||
if (separator <= 0 || separator === output.length - 1) {
|
||||
throw new Error('No active chat-scoped Gateway API Key with a positive balance was found');
|
||||
}
|
||||
return { keyId: output.slice(0, separator), value: output.slice(separator + 1), source: 'database' };
|
||||
}
|
||||
|
||||
async function fetchText(path, options = {}) {
|
||||
const response = await fetch(`${baseURL}${path}`, {
|
||||
...options,
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
headers: { Authorization: `Bearer ${authentication.value}`, ...(options.headers || {}) },
|
||||
});
|
||||
const text = await response.text();
|
||||
let body = null;
|
||||
try {
|
||||
body = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
body = { raw: text.slice(0, 2_000) };
|
||||
}
|
||||
return { response, body };
|
||||
}
|
||||
|
||||
async function getTask(taskId) {
|
||||
const { response, body } = await fetchText(`/api/v1/tasks/${taskId}`);
|
||||
if (!response.ok) throw new Error(`GET task ${taskId} failed ${response.status}: ${JSON.stringify(body)}`);
|
||||
return body;
|
||||
}
|
||||
|
||||
async function getPreprocessing(taskId) {
|
||||
const { response, body } = await fetchText(`/api/v1/tasks/${taskId}/param-preprocessing`);
|
||||
if (!response.ok) throw new Error(`GET preprocessing ${taskId} failed ${response.status}: ${JSON.stringify(body)}`);
|
||||
return body?.items || [];
|
||||
}
|
||||
|
||||
function successfulAttempt(task) {
|
||||
const attempts = Array.isArray(task.attempts) ? task.attempts : [];
|
||||
return attempts.findLast((attempt) => attempt.status === 'succeeded') || attempts.at(-1);
|
||||
}
|
||||
|
||||
function hasOwn(value, key) {
|
||||
return value != null && Object.prototype.hasOwnProperty.call(value, key);
|
||||
}
|
||||
|
||||
function validateAudit(scenario, task, preprocessing) {
|
||||
const attempt = successfulAttempt(task);
|
||||
if (!attempt) throw new Error('task does not contain an upstream attempt');
|
||||
if (task.simulated || attempt.simulated) throw new Error('request was served by a simulation candidate');
|
||||
const platformName = String(attempt.metrics?.platformName || task.metrics?.platformName || '');
|
||||
if (!platformName.includes(scenario.platform)) {
|
||||
throw new Error(`unexpected platform ${platformName}; expected ${scenario.platform}`);
|
||||
}
|
||||
const snapshot = attempt.requestSnapshot || {};
|
||||
const expected = scenario.expected;
|
||||
if (expected.absent) {
|
||||
if (hasOwn(snapshot, expected.parameter)) {
|
||||
throw new Error(`${expected.parameter} was unexpectedly sent upstream as ${JSON.stringify(snapshot[expected.parameter])}`);
|
||||
}
|
||||
} else if (snapshot[expected.parameter] !== expected.value) {
|
||||
throw new Error(`${expected.parameter}=${JSON.stringify(snapshot[expected.parameter])}; expected ${expected.value}`);
|
||||
}
|
||||
for (const [key, value] of Object.entries(expected.preserved || {})) {
|
||||
if (snapshot[key] !== value) throw new Error(`${key} was not preserved in the upstream snapshot`);
|
||||
}
|
||||
const changes = preprocessing.flatMap((item) => item.changes || []);
|
||||
const outputLimitChanges = changes.filter((change) => change.processor === 'OutputTokenLimitProcessor');
|
||||
if (expected.injected) {
|
||||
const matched = outputLimitChanges.some((change) => change.action === 'set' && change.path === expected.parameter && change.after === expected.value);
|
||||
if (!matched) throw new Error(`missing OutputTokenLimitProcessor audit for ${expected.parameter}=${expected.value}`);
|
||||
} else if (outputLimitChanges.length > 0) {
|
||||
throw new Error(`unexpected OutputTokenLimitProcessor audit: ${JSON.stringify(outputLimitChanges)}`);
|
||||
}
|
||||
return {
|
||||
taskId: task.id,
|
||||
attemptId: attempt.id,
|
||||
platform: platformName,
|
||||
provider: attempt.metrics?.provider || task.metrics?.provider || null,
|
||||
platformModelId: attempt.metrics?.platformModelId || task.metrics?.platformModelId || null,
|
||||
upstreamProtocol: attempt.metrics?.upstreamProtocol || task.metrics?.upstreamProtocol || null,
|
||||
upstreamEndpoint: attempt.metrics?.upstreamEndpoint || task.metrics?.upstreamEndpoint || null,
|
||||
simulated: Boolean(task.simulated || attempt.simulated),
|
||||
requestParameters: {
|
||||
max_tokens: hasOwn(snapshot, 'max_tokens') ? snapshot.max_tokens : '(absent)',
|
||||
max_completion_tokens: hasOwn(snapshot, 'max_completion_tokens') ? snapshot.max_completion_tokens : '(absent)',
|
||||
max_output_tokens: hasOwn(snapshot, 'max_output_tokens') ? snapshot.max_output_tokens : '(absent)',
|
||||
},
|
||||
outputLimitAudit: outputLimitChanges,
|
||||
usage: task.usage || attempt.usage || null,
|
||||
};
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const scenario of scenarios) {
|
||||
const startedAt = Date.now();
|
||||
let taskId = null;
|
||||
try {
|
||||
const requestBody = { model: scenario.model, ...scenario.body };
|
||||
const { response, body } = await fetchText(scenario.path, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestBody),
|
||||
});
|
||||
taskId = response.headers.get('x-gateway-task-id');
|
||||
if (!response.ok) throw new Error(`POST ${scenario.path} failed ${response.status}: ${JSON.stringify(body)}`);
|
||||
if (!taskId) throw new Error('response is missing X-Gateway-Task-Id');
|
||||
const [task, preprocessing] = await Promise.all([getTask(taskId), getPreprocessing(taskId)]);
|
||||
const audit = validateAudit(scenario, task, preprocessing);
|
||||
results.push({ name: scenario.name, model: scenario.model, path: scenario.path, passed: true, durationMs: Date.now() - startedAt, audit });
|
||||
process.stderr.write(`PASS ${scenario.name} (${Date.now() - startedAt} ms)\n`);
|
||||
} catch (error) {
|
||||
results.push({ name: scenario.name, model: scenario.model, path: scenario.path, passed: false, durationMs: Date.now() - startedAt, taskId, error: error instanceof Error ? error.message : String(error) });
|
||||
process.stderr.write(`FAIL ${scenario.name}: ${error instanceof Error ? error.message : String(error)}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
const report = {
|
||||
ok: results.every((result) => result.passed),
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseURL,
|
||||
authentication: { type: 'gateway_api_key', source: authentication.source, keyId: authentication.keyId },
|
||||
summary: { total: results.length, passed: results.filter((result) => result.passed).length, failed: results.filter((result) => !result.passed).length },
|
||||
results,
|
||||
};
|
||||
const outputPath = process.env.GATEWAY_E2E_OUTPUT;
|
||||
if (outputPath) {
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
}
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
if (!report.ok) process.exitCode = 1;
|
||||
Reference in New Issue
Block a user