273 lines
12 KiB
JavaScript
273 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { execFileSync } from 'node:child_process';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { dirname } from 'node:path';
|
|
|
|
const baseURL = (process.env.GATEWAY_BASE_URL || 'http://127.0.0.1:8088').replace(/\/+$/, '');
|
|
const authentication = resolveGatewayAPIKey();
|
|
const timeoutMs = Number(process.env.GATEWAY_E2E_TIMEOUT_MS || 180_000);
|
|
|
|
const scenarios = [
|
|
{
|
|
name: '火山 128K Chat 未传上限',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', value: 43690, injected: true },
|
|
},
|
|
{
|
|
name: '火山 32K Chat 未传上限',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_32K_MODEL || 'volces-openai:doubao-seed-1-8-251228',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', value: 10922, injected: true },
|
|
},
|
|
{
|
|
name: '火山 Chat max_tokens=null',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody({ max_tokens: null }),
|
|
expected: { parameter: 'max_tokens', value: 43690, injected: true },
|
|
},
|
|
{
|
|
name: '火山 Chat 显式 max_tokens',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody({ max_tokens: 64 }),
|
|
expected: { parameter: 'max_tokens', value: 64, injected: false },
|
|
},
|
|
{
|
|
name: '火山 Chat 显式 max_completion_tokens',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody({ max_completion_tokens: 64 }),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false, preserved: { max_completion_tokens: 64 } },
|
|
},
|
|
{
|
|
name: '火山 Responses 未传上限',
|
|
platform: '火山引擎',
|
|
model: process.env.GATEWAY_E2E_VOLCES_128K_MODEL || 'volces-openai:doubao-seed-2-0-pro-260215',
|
|
path: '/v1/responses',
|
|
body: responsesBody(),
|
|
expected: { parameter: 'max_output_tokens', value: 43690, injected: true },
|
|
},
|
|
{
|
|
name: '阿里百炼 Qwen Chat 未传上限',
|
|
platform: '阿里云百炼',
|
|
model: process.env.GATEWAY_E2E_QWEN_MODEL || 'aliyun-bailian-openai:qwen3.7-plus',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: '阿里百炼 Qwen Responses 未传上限',
|
|
platform: '阿里云百炼',
|
|
model: process.env.GATEWAY_E2E_QWEN_MODEL || 'aliyun-bailian-openai:qwen3.7-plus',
|
|
path: '/v1/responses',
|
|
body: responsesBody(),
|
|
expected: { parameter: 'max_output_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: 'DeepSeek 官网 Chat 未传上限',
|
|
platform: 'DeepSeek 官网',
|
|
model: process.env.GATEWAY_E2E_DEEPSEEK_MODEL || 'deepseek-openai:deepseek-v4-pro',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: 'DeepSeek 官网 Responses 回退未传上限',
|
|
platform: 'DeepSeek 官网',
|
|
model: process.env.GATEWAY_E2E_DEEPSEEK_MODEL || 'deepseek-openai:deepseek-v4-pro',
|
|
path: '/v1/responses',
|
|
body: responsesBody(),
|
|
expected: { parameter: 'max_output_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: '智谱 GLM-5.2 Chat 未传上限',
|
|
platform: '智谱AI',
|
|
model: process.env.GATEWAY_E2E_ZHIPU_MODEL || 'zhipu-openai:glm-5.2',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: 'OpenAI GPT-4o Chat 未传上限',
|
|
platform: 'OpenAI',
|
|
model: process.env.GATEWAY_E2E_OPENAI_MODEL || 'openai:gpt-4o',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
|
},
|
|
{
|
|
name: 'Google Gemini Chat 未传上限',
|
|
platform: 'Google Gemini',
|
|
model: process.env.GATEWAY_E2E_GEMINI_MODEL || 'gemini:gemini-2.5-pro',
|
|
path: '/v1/chat/completions',
|
|
body: chatBody(),
|
|
expected: { parameter: 'max_tokens', absent: true, injected: false },
|
|
},
|
|
];
|
|
|
|
function chatBody(extra = {}) {
|
|
return {
|
|
messages: [{ role: 'user', content: '这是输出上限链路验证。只回复 OK。' }],
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
function responsesBody(extra = {}) {
|
|
return { input: '这是输出上限链路验证。只回复 OK。', ...extra };
|
|
}
|
|
|
|
function resolveGatewayAPIKey() {
|
|
if (process.env.GATEWAY_E2E_API_KEY) {
|
|
return { value: process.env.GATEWAY_E2E_API_KEY, source: 'environment', keyId: null };
|
|
}
|
|
const container = process.env.GATEWAY_E2E_DB_CONTAINER || 'postgres';
|
|
const database = process.env.GATEWAY_E2E_DB_NAME || 'easyai_ai_gateway';
|
|
const user = process.env.GATEWAY_E2E_DB_USER || 'easyai';
|
|
const query = `SELECT key.id::text || E'\\t' || key.key_secret FROM gateway_api_keys key JOIN gateway_wallet_accounts wallet ON wallet.gateway_user_id = key.gateway_user_id AND wallet.status = 'active' WHERE key.deleted_at IS NULL AND key.status = 'active' AND key.key_secret IS NOT NULL AND key.key_secret <> '' AND key.scopes ? 'chat' AND (key.expires_at IS NULL OR key.expires_at > now()) AND (wallet.balance - wallet.frozen_balance) > 0 ORDER BY (wallet.balance - wallet.frozen_balance) DESC, key.created_at DESC LIMIT 1`;
|
|
let output = '';
|
|
try {
|
|
output = execFileSync('docker', ['exec', container, 'psql', '-U', user, '-d', database, '-At', '-c', query], { encoding: 'utf8' }).trim();
|
|
} catch {
|
|
throw new Error('Unable to load an active chat-scoped Gateway API Key; set GATEWAY_E2E_API_KEY explicitly');
|
|
}
|
|
const separator = output.indexOf('\t');
|
|
if (separator <= 0 || separator === output.length - 1) {
|
|
throw new Error('No active chat-scoped Gateway API Key with a positive balance was found');
|
|
}
|
|
return { keyId: output.slice(0, separator), value: output.slice(separator + 1), source: 'database' };
|
|
}
|
|
|
|
async function fetchText(path, options = {}) {
|
|
const response = await fetch(`${baseURL}${path}`, {
|
|
...options,
|
|
signal: AbortSignal.timeout(timeoutMs),
|
|
headers: { Authorization: `Bearer ${authentication.value}`, ...(options.headers || {}) },
|
|
});
|
|
const text = await response.text();
|
|
let body = null;
|
|
try {
|
|
body = text ? JSON.parse(text) : null;
|
|
} catch {
|
|
body = { raw: text.slice(0, 2_000) };
|
|
}
|
|
return { response, body };
|
|
}
|
|
|
|
async function getTask(taskId) {
|
|
const { response, body } = await fetchText(`/api/v1/tasks/${taskId}`);
|
|
if (!response.ok) throw new Error(`GET task ${taskId} failed ${response.status}: ${JSON.stringify(body)}`);
|
|
return body;
|
|
}
|
|
|
|
async function getPreprocessing(taskId) {
|
|
const { response, body } = await fetchText(`/api/v1/tasks/${taskId}/param-preprocessing`);
|
|
if (!response.ok) throw new Error(`GET preprocessing ${taskId} failed ${response.status}: ${JSON.stringify(body)}`);
|
|
return body?.items || [];
|
|
}
|
|
|
|
function successfulAttempt(task) {
|
|
const attempts = Array.isArray(task.attempts) ? task.attempts : [];
|
|
return attempts.findLast((attempt) => attempt.status === 'succeeded') || attempts.at(-1);
|
|
}
|
|
|
|
function hasOwn(value, key) {
|
|
return value != null && Object.prototype.hasOwnProperty.call(value, key);
|
|
}
|
|
|
|
function validateAudit(scenario, task, preprocessing) {
|
|
const attempt = successfulAttempt(task);
|
|
if (!attempt) throw new Error('task does not contain an upstream attempt');
|
|
if (task.simulated || attempt.simulated) throw new Error('request was served by a simulation candidate');
|
|
const platformName = String(attempt.metrics?.platformName || task.metrics?.platformName || '');
|
|
if (!platformName.includes(scenario.platform)) {
|
|
throw new Error(`unexpected platform ${platformName}; expected ${scenario.platform}`);
|
|
}
|
|
const snapshot = attempt.requestSnapshot || {};
|
|
const expected = scenario.expected;
|
|
if (expected.absent) {
|
|
if (hasOwn(snapshot, expected.parameter)) {
|
|
throw new Error(`${expected.parameter} was unexpectedly sent upstream as ${JSON.stringify(snapshot[expected.parameter])}`);
|
|
}
|
|
} else if (snapshot[expected.parameter] !== expected.value) {
|
|
throw new Error(`${expected.parameter}=${JSON.stringify(snapshot[expected.parameter])}; expected ${expected.value}`);
|
|
}
|
|
for (const [key, value] of Object.entries(expected.preserved || {})) {
|
|
if (snapshot[key] !== value) throw new Error(`${key} was not preserved in the upstream snapshot`);
|
|
}
|
|
const changes = preprocessing.flatMap((item) => item.changes || []);
|
|
const outputLimitChanges = changes.filter((change) => change.processor === 'OutputTokenLimitProcessor');
|
|
if (expected.injected) {
|
|
const matched = outputLimitChanges.some((change) => change.action === 'set' && change.path === expected.parameter && change.after === expected.value);
|
|
if (!matched) throw new Error(`missing OutputTokenLimitProcessor audit for ${expected.parameter}=${expected.value}`);
|
|
} else if (outputLimitChanges.length > 0) {
|
|
throw new Error(`unexpected OutputTokenLimitProcessor audit: ${JSON.stringify(outputLimitChanges)}`);
|
|
}
|
|
return {
|
|
taskId: task.id,
|
|
attemptId: attempt.id,
|
|
platform: platformName,
|
|
provider: attempt.metrics?.provider || task.metrics?.provider || null,
|
|
platformModelId: attempt.metrics?.platformModelId || task.metrics?.platformModelId || null,
|
|
upstreamProtocol: attempt.metrics?.upstreamProtocol || task.metrics?.upstreamProtocol || null,
|
|
upstreamEndpoint: attempt.metrics?.upstreamEndpoint || task.metrics?.upstreamEndpoint || null,
|
|
simulated: Boolean(task.simulated || attempt.simulated),
|
|
requestParameters: {
|
|
max_tokens: hasOwn(snapshot, 'max_tokens') ? snapshot.max_tokens : '(absent)',
|
|
max_completion_tokens: hasOwn(snapshot, 'max_completion_tokens') ? snapshot.max_completion_tokens : '(absent)',
|
|
max_output_tokens: hasOwn(snapshot, 'max_output_tokens') ? snapshot.max_output_tokens : '(absent)',
|
|
},
|
|
outputLimitAudit: outputLimitChanges,
|
|
usage: task.usage || attempt.usage || null,
|
|
};
|
|
}
|
|
|
|
const results = [];
|
|
for (const scenario of scenarios) {
|
|
const startedAt = Date.now();
|
|
let taskId = null;
|
|
try {
|
|
const requestBody = { model: scenario.model, ...scenario.body };
|
|
const { response, body } = await fetchText(scenario.path, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(requestBody),
|
|
});
|
|
taskId = response.headers.get('x-gateway-task-id');
|
|
if (!response.ok) throw new Error(`POST ${scenario.path} failed ${response.status}: ${JSON.stringify(body)}`);
|
|
if (!taskId) throw new Error('response is missing X-Gateway-Task-Id');
|
|
const [task, preprocessing] = await Promise.all([getTask(taskId), getPreprocessing(taskId)]);
|
|
const audit = validateAudit(scenario, task, preprocessing);
|
|
results.push({ name: scenario.name, model: scenario.model, path: scenario.path, passed: true, durationMs: Date.now() - startedAt, audit });
|
|
process.stderr.write(`PASS ${scenario.name} (${Date.now() - startedAt} ms)\n`);
|
|
} catch (error) {
|
|
results.push({ name: scenario.name, model: scenario.model, path: scenario.path, passed: false, durationMs: Date.now() - startedAt, taskId, error: error instanceof Error ? error.message : String(error) });
|
|
process.stderr.write(`FAIL ${scenario.name}: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
}
|
|
}
|
|
|
|
const report = {
|
|
ok: results.every((result) => result.passed),
|
|
generatedAt: new Date().toISOString(),
|
|
baseURL,
|
|
authentication: { type: 'gateway_api_key', source: authentication.source, keyId: authentication.keyId },
|
|
summary: { total: results.length, passed: results.filter((result) => result.passed).length, failed: results.filter((result) => !result.passed).length },
|
|
results,
|
|
};
|
|
const outputPath = process.env.GATEWAY_E2E_OUTPUT;
|
|
if (outputPath) {
|
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
}
|
|
console.log(JSON.stringify(report, null, 2));
|
|
if (!report.ok) process.exitCode = 1;
|