260 lines
12 KiB
JavaScript
260 lines
12 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { dirname } from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
|
|
const baseURL = (process.env.GATEWAY_BASE_URL || 'http://127.0.0.1:8088').replace(/\/+$/, '');
|
|
const authentication = resolveGatewayAPIKey();
|
|
|
|
const modelCases = [
|
|
{
|
|
model: process.env.GATEWAY_E2E_DOUBAO_MODEL || 'Doubao Seed 2.0 Pro',
|
|
platform: '火山',
|
|
protocol: 'openai_responses',
|
|
converted: false,
|
|
},
|
|
{
|
|
model: process.env.GATEWAY_E2E_QWEN_MODEL || 'Qwen3.7-Plus',
|
|
platform: '百炼',
|
|
protocol: 'openai_responses',
|
|
converted: false,
|
|
},
|
|
{
|
|
model: process.env.GATEWAY_E2E_DEEPSEEK_MODEL || 'deepseek-openai:deepseek-v4-pro',
|
|
platform: 'DeepSeek 官网',
|
|
protocol: 'openai_chat_completions',
|
|
converted: true,
|
|
},
|
|
];
|
|
|
|
function assert(condition, message) {
|
|
if (!condition) throw new Error(message);
|
|
}
|
|
|
|
function resolveGatewayAPIKey() {
|
|
if (process.env.GATEWAY_E2E_API_KEY) {
|
|
return { value: process.env.GATEWAY_E2E_API_KEY, source: 'environment', keyId: null };
|
|
}
|
|
const container = process.env.GATEWAY_E2E_DB_CONTAINER || 'postgres';
|
|
const database = process.env.GATEWAY_E2E_DB_NAME || 'easyai_ai_gateway';
|
|
const user = process.env.GATEWAY_E2E_DB_USER || 'easyai';
|
|
const query = `SELECT key.id::text || E'\\t' || key.key_secret FROM gateway_api_keys key JOIN gateway_wallet_accounts wallet ON wallet.gateway_user_id = key.gateway_user_id AND wallet.status = 'active' WHERE key.deleted_at IS NULL AND key.status = 'active' AND key.key_secret IS NOT NULL AND key.key_secret <> '' AND key.scopes ? 'chat' AND (key.expires_at IS NULL OR key.expires_at > now()) AND (wallet.balance - wallet.frozen_balance) > 0 ORDER BY (wallet.balance - wallet.frozen_balance) DESC, key.created_at DESC LIMIT 1`;
|
|
let output = '';
|
|
try {
|
|
output = execFileSync('docker', ['exec', container, 'psql', '-U', user, '-d', database, '-At', '-c', query], { encoding: 'utf8' }).trim();
|
|
} catch {
|
|
throw new Error('Unable to load an active chat-scoped Gateway API Key from the database; set GATEWAY_E2E_API_KEY explicitly');
|
|
}
|
|
const separator = output.indexOf('\t');
|
|
if (separator <= 0 || separator === output.length - 1) {
|
|
throw new Error('The database does not contain an active chat-scoped Gateway API Key with a stored secret');
|
|
}
|
|
return { keyId: output.slice(0, separator), value: output.slice(separator + 1), source: 'database' };
|
|
}
|
|
|
|
async function requestJSON(token, path, body) {
|
|
const response = await fetch(`${baseURL}${path}`, {
|
|
method: 'POST',
|
|
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(body),
|
|
});
|
|
const text = await response.text();
|
|
if (!response.ok) throw new Error(`POST ${path} failed ${response.status}: ${text}`);
|
|
return { body: JSON.parse(text), taskId: response.headers.get('x-gateway-task-id') };
|
|
}
|
|
|
|
async function requestStream(token, path, body) {
|
|
const response = await fetch(`${baseURL}${path}`, {
|
|
method: 'POST',
|
|
headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ ...body, stream: true }),
|
|
});
|
|
const text = await response.text();
|
|
if (!response.ok) throw new Error(`POST ${path} failed ${response.status}: ${text}`);
|
|
const events = parseSSE(text);
|
|
const errorEvent = events.find((event) => event.event === 'error');
|
|
if (errorEvent) throw new Error(`Responses stream failed: ${JSON.stringify(errorEvent.data)}`);
|
|
const completed = events.findLast((event) => event.event === 'response.completed');
|
|
assert(completed?.data?.response, `stream is missing response.completed: ${text}`);
|
|
validateEventOrder(events);
|
|
return { events, body: completed.data.response, taskId: response.headers.get('x-gateway-task-id') };
|
|
}
|
|
|
|
function parseSSE(text) {
|
|
return text.split(/\r?\n\r?\n/).flatMap((block) => {
|
|
if (!block.trim()) return [];
|
|
let event = 'message';
|
|
const data = [];
|
|
for (const line of block.split(/\r?\n/)) {
|
|
if (line.startsWith('event:')) event = line.slice(6).trim();
|
|
if (line.startsWith('data:')) data.push(line.slice(5).trim());
|
|
}
|
|
if (!data.length) return [];
|
|
return [{ event, data: JSON.parse(data.join('\n')) }];
|
|
});
|
|
}
|
|
|
|
function validateEventOrder(events) {
|
|
const names = events.map((event) => event.event);
|
|
const created = names.indexOf('response.created');
|
|
const inProgress = names.indexOf('response.in_progress');
|
|
const completed = names.lastIndexOf('response.completed');
|
|
assert(created >= 0, `missing response.created: ${names.join(', ')}`);
|
|
assert(inProgress > created, `response.in_progress is out of order: ${names.join(', ')}`);
|
|
assert(completed > inProgress, `response.completed is out of order: ${names.join(', ')}`);
|
|
assert(completed === names.length - 1, `response.completed must be the final event: ${names.join(', ')}`);
|
|
assert(!names.includes('[DONE]'), 'Responses stream must not contain a Chat [DONE] event');
|
|
}
|
|
|
|
function responseText(response) {
|
|
if (typeof response.output_text === 'string') return response.output_text;
|
|
return (response.output || [])
|
|
.flatMap((item) => item.type === 'message' ? item.content || [] : [])
|
|
.filter((item) => item.type === 'output_text')
|
|
.map((item) => item.text || '')
|
|
.join('');
|
|
}
|
|
|
|
function functionCall(response) {
|
|
return (response.output || []).find((item) => item.type === 'function_call');
|
|
}
|
|
|
|
async function getTask(token, taskId) {
|
|
assert(taskId, 'response is missing X-Gateway-Task-Id');
|
|
const response = await fetch(`${baseURL}/api/v1/tasks/${taskId}`, {
|
|
headers: { Authorization: `Bearer ${token}` },
|
|
});
|
|
const text = await response.text();
|
|
if (!response.ok) throw new Error(`GET task failed ${response.status}: ${text}`);
|
|
return JSON.parse(text);
|
|
}
|
|
|
|
function auditSummary(task, expected) {
|
|
const metrics = task.metrics || {};
|
|
assert(String(metrics.platformName || '').includes(expected.platform), `${expected.model} used unexpected platform: ${metrics.platformName}`);
|
|
assert(metrics.upstreamProtocol === expected.protocol, `${expected.model} used ${metrics.upstreamProtocol}, expected ${expected.protocol}`);
|
|
assert(Boolean(metrics.responseConverted) === expected.converted, `${expected.model} responseConverted mismatch`);
|
|
assert(metrics.upstreamEndpoint === (expected.converted ? '/chat/completions' : '/responses'), `${expected.model} upstream endpoint mismatch`);
|
|
if (expected.converted) {
|
|
assert(metrics.publicResponseId === `resp_${String(task.id).replaceAll('-', '')}`, `${expected.model} Chat fallback id is not derived from the task id`);
|
|
} else {
|
|
assert(metrics.publicResponseId === metrics.upstreamResponseId, `${expected.model} native Responses id was not preserved`);
|
|
}
|
|
return {
|
|
taskId: task.id,
|
|
publicResponseId: metrics.publicResponseId,
|
|
upstreamResponseId: metrics.upstreamResponseId,
|
|
platform: metrics.platformName,
|
|
platformModelId: metrics.platformModelId,
|
|
protocol: metrics.upstreamProtocol,
|
|
upstreamEndpoint: metrics.upstreamEndpoint,
|
|
responseConverted: metrics.responseConverted,
|
|
parentResponseId: metrics.parentResponseId || null,
|
|
chainDepth: metrics.responseChainDepth,
|
|
usage: task.usage,
|
|
finalChargeAmount: task.finalChargeAmount,
|
|
billingSummary: task.billingSummary,
|
|
billings: task.billings,
|
|
};
|
|
}
|
|
|
|
const token = authentication.value;
|
|
const results = [];
|
|
|
|
for (const expected of modelCases) {
|
|
const nonce = `E2E-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 9)}`;
|
|
const first = await requestJSON(token, '/v1/responses', {
|
|
model: expected.model,
|
|
input: `请记住校验码 ${nonce}。只回复“已记住”。`,
|
|
store: true,
|
|
});
|
|
assert(/^resp_[A-Za-z0-9_-]+$/.test(first.body.id), `${expected.model} returned invalid response id`);
|
|
const second = await requestJSON(token, '/v1/responses', {
|
|
model: expected.model,
|
|
input: '我刚才让你记住的校验码是什么？只回复校验码。',
|
|
previous_response_id: first.body.id,
|
|
store: true,
|
|
});
|
|
assert(second.body.previous_response_id === first.body.id, `${expected.model} response did not preserve previous_response_id`);
|
|
assert(responseText(second.body).includes(nonce), `${expected.model} did not preserve ordinary conversation continuity`);
|
|
|
|
const toolOutput = `TOOL-${nonce}`;
|
|
const normalizedModel = expected.model.toLowerCase();
|
|
const requiresNonThinkingToolMode = normalizedModel.startsWith('qwen3.7-') || normalizedModel.includes('deepseek-v4-pro');
|
|
const toolReasoning = requiresNonThinkingToolMode ? { reasoning: { effort: 'none' } } : {};
|
|
const forcedToolChoice = 'required';
|
|
const toolFirst = await requestStream(token, '/v1/responses', {
|
|
model: expected.model,
|
|
input: '调用 lookup_verification_code 获取校验结果。',
|
|
tools: [{
|
|
type: 'function',
|
|
name: 'lookup_verification_code',
|
|
description: '返回测试校验结果',
|
|
parameters: { type: 'object', properties: { scope: { type: 'string' } }, required: ['scope'], additionalProperties: false },
|
|
strict: true,
|
|
}],
|
|
tool_choice: forcedToolChoice,
|
|
parallel_tool_calls: true,
|
|
...toolReasoning,
|
|
store: true,
|
|
});
|
|
const call = functionCall(toolFirst.body);
|
|
assert(call?.call_id, `${expected.model} tool response is missing call_id`);
|
|
assert(toolFirst.events.some((event) => event.event === 'response.function_call_arguments.delta'), `${expected.model} stream is missing function arguments delta`);
|
|
|
|
const toolSecond = await requestStream(token, '/v1/responses', {
|
|
model: expected.model,
|
|
previous_response_id: toolFirst.body.id,
|
|
input: [{ type: 'function_call_output', call_id: call.call_id, output: JSON.stringify({ verification: toolOutput }) }],
|
|
...toolReasoning,
|
|
store: true,
|
|
});
|
|
assert(responseText(toolSecond.body).includes(toolOutput), `${expected.model} did not consume function_call_output`);
|
|
|
|
const auditedTasks = await Promise.all([
|
|
getTask(token, first.taskId),
|
|
getTask(token, second.taskId),
|
|
getTask(token, toolFirst.taskId),
|
|
getTask(token, toolSecond.taskId),
|
|
]);
|
|
const ordinaryFirstAudit = auditSummary(auditedTasks[0], expected);
|
|
const ordinarySecondAudit = auditSummary(auditedTasks[1], expected);
|
|
const toolFirstAudit = auditSummary(auditedTasks[2], expected);
|
|
const toolSecondAudit = auditSummary(auditedTasks[3], expected);
|
|
assert(ordinarySecondAudit.parentResponseId === first.body.id, `${expected.model} ordinary parent response id mismatch`);
|
|
assert(ordinarySecondAudit.chainDepth === 1, `${expected.model} ordinary chain depth mismatch`);
|
|
assert(toolSecondAudit.parentResponseId === toolFirst.body.id, `${expected.model} tool parent response id mismatch`);
|
|
assert(toolSecondAudit.chainDepth === 1, `${expected.model} tool chain depth mismatch`);
|
|
results.push({
|
|
model: expected.model,
|
|
ordinaryConversation: {
|
|
passed: true,
|
|
first: ordinaryFirstAudit,
|
|
second: ordinarySecondAudit,
|
|
},
|
|
toolCalling: {
|
|
passed: true,
|
|
callId: call.call_id,
|
|
first: toolFirstAudit,
|
|
second: toolSecondAudit,
|
|
firstEventTypes: toolFirst.events.map((event) => event.event),
|
|
secondEventTypes: toolSecond.events.map((event) => event.event),
|
|
},
|
|
});
|
|
}
|
|
|
|
const report = {
|
|
ok: true,
|
|
generatedAt: new Date().toISOString(),
|
|
baseURL,
|
|
authentication: { type: 'gateway_api_key', source: authentication.source, keyId: authentication.keyId },
|
|
results,
|
|
};
|
|
const outputPath = process.env.GATEWAY_E2E_OUTPUT;
|
|
if (outputPath) {
|
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
|
}
|
|
console.log(JSON.stringify(report, null, 2));
|