test(e2e): 增加视频生产验收脚本与记录
ci / verify (pull_request) Failing after 3m43s
ci / verify (pull_request) Failing after 3m43s
This commit is contained in:
@@ -0,0 +1,634 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const baseURL = (process.env.GATEWAY_BASE_URL || 'http://127.0.0.1:8088').replace(/\/+$/, '');
|
||||
const platformKey = process.env.KELING_VIDEO_PLATFORM_KEY || 'transtreams_keling';
|
||||
const pollIntervalMs = positiveInteger(process.env.KELING_VIDEO_POLL_INTERVAL_MS, 10_000);
|
||||
const taskTimeoutMs = positiveInteger(process.env.KELING_VIDEO_TASK_TIMEOUT_MS, 30 * 60 * 1000);
|
||||
const outputPath =
|
||||
process.env.KELING_VIDEO_E2E_OUTPUT ||
|
||||
`artifacts/keling-video-e2e-${new Date().toISOString().replaceAll(/[:.]/g, '-')}.json`;
|
||||
const database = {
|
||||
container: process.env.GATEWAY_E2E_DB_CONTAINER || 'postgres',
|
||||
name: process.env.GATEWAY_E2E_DB_NAME || 'easyai_ai_gateway',
|
||||
user: process.env.GATEWAY_E2E_DB_USER || 'easyai',
|
||||
};
|
||||
|
||||
const validationCases = [
|
||||
{
|
||||
name: 'compatible-o1-720p-16x9-3s-audio-off',
|
||||
interface: 'kling-compatible',
|
||||
requestedModel: 'kling-video-o1',
|
||||
gatewayModel: 'kling-o1',
|
||||
providerModel: 'kling-video-o1',
|
||||
request: {
|
||||
model_name: 'kling-video-o1',
|
||||
prompt: 'A blue ceramic cup rotates slowly on a clean studio table, locked camera, no sound.',
|
||||
mode: 'std',
|
||||
aspect_ratio: '16:9',
|
||||
duration: '3',
|
||||
sound: 'off',
|
||||
image_list: [
|
||||
{
|
||||
image_url: 'https://placehold.co/1024x1024/png',
|
||||
},
|
||||
],
|
||||
watermark_info: { enabled: false },
|
||||
external_task_id: 'gateway-keling-e2e-compatible-o1',
|
||||
},
|
||||
expected: { resolution: '720p', shortEdge: 720, ratio: 16 / 9, duration: 3, audio: false },
|
||||
},
|
||||
{
|
||||
name: 'compatible-v3-1080p-9x16-5s-audio-on',
|
||||
interface: 'kling-compatible',
|
||||
requestedModel: 'kling-v3-omni',
|
||||
gatewayModel: 'kling-3.0-omni',
|
||||
providerModel: 'kling-v3-omni',
|
||||
request: {
|
||||
model_name: 'kling-v3-omni',
|
||||
prompt: 'Vertical close-up of gentle rain falling on green leaves, with clear natural rain ambience.',
|
||||
mode: 'pro',
|
||||
aspect_ratio: '9:16',
|
||||
duration: 5,
|
||||
sound: 'on',
|
||||
watermark_info: { enabled: false },
|
||||
external_task_id: 'gateway-keling-e2e-compatible-v3',
|
||||
},
|
||||
expected: { resolution: '1080p', shortEdge: 1080, ratio: 9 / 16, duration: 5, audio: true },
|
||||
},
|
||||
{
|
||||
name: 'standard-o1-1080p-9x16-5s-audio-on',
|
||||
interface: 'gateway-standard',
|
||||
requestedModel: 'kling-o1',
|
||||
gatewayModel: 'kling-o1',
|
||||
providerModel: 'kling-video-o1',
|
||||
request: {
|
||||
model: 'kling-o1',
|
||||
prompt: 'Vertical view of small ocean waves reaching a sandy beach, with audible natural surf.',
|
||||
resolution: '1080p',
|
||||
aspect_ratio: '9:16',
|
||||
duration: 5,
|
||||
audio: true,
|
||||
watermark: false,
|
||||
runMode: 'real',
|
||||
},
|
||||
expected: { resolution: '1080p', shortEdge: 1080, ratio: 9 / 16, duration: 5, audio: true },
|
||||
},
|
||||
{
|
||||
name: 'standard-v3-720p-16x9-3s-audio-off',
|
||||
interface: 'gateway-standard',
|
||||
requestedModel: 'kling-3.0-omni',
|
||||
gatewayModel: 'kling-3.0-omni',
|
||||
providerModel: 'kling-v3-omni',
|
||||
request: {
|
||||
model: 'kling-3.0-omni',
|
||||
prompt: 'Wide shot of a paper windmill turning slowly beside a window, locked camera, no sound.',
|
||||
resolution: '720p',
|
||||
aspect_ratio: '16:9',
|
||||
duration: 3,
|
||||
audio: false,
|
||||
watermark: false,
|
||||
runMode: 'real',
|
||||
},
|
||||
expected: { resolution: '720p', shortEdge: 720, ratio: 16 / 9, duration: 3, audio: false },
|
||||
},
|
||||
];
|
||||
|
||||
const selectedNames = new Set(
|
||||
String(process.env.KELING_VIDEO_CASES || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
const resumeTaskIds = parseResumeTaskIds(process.env.KELING_VIDEO_RESUME_TASKS);
|
||||
const continueOnValidationFailure = process.env.KELING_VIDEO_CONTINUE_ON_VALIDATION_FAILURE === 'true';
|
||||
if (selectedNames.size > 0) {
|
||||
const selected = validationCases.filter((validationCase) => selectedNames.has(validationCase.name));
|
||||
if (selected.length !== selectedNames.size) {
|
||||
throw new Error(`Unknown KELING_VIDEO_CASES value; known cases: ${validationCases.map((item) => item.name).join(', ')}`);
|
||||
}
|
||||
validationCases.splice(0, validationCases.length, ...selected);
|
||||
}
|
||||
|
||||
const submittedTasks = [];
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
|
||||
function positiveInteger(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value || ''), 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function parseResumeTaskIds(raw) {
|
||||
if (!String(raw || '').trim()) return {};
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (error) {
|
||||
throw new Error(`KELING_VIDEO_RESUME_TASKS must be a JSON object: ${error.message}`);
|
||||
}
|
||||
assert(parsed && typeof parsed === 'object' && !Array.isArray(parsed), 'KELING_VIDEO_RESUME_TASKS must be a JSON object');
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function nearlyEqual(left, right, tolerance = 1e-6) {
|
||||
return Math.abs(Number(left) - Number(right)) <= tolerance;
|
||||
}
|
||||
|
||||
function sqlLiteral(value) {
|
||||
return `'${String(value).replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function runPSQL(query) {
|
||||
return execFileSync(
|
||||
'docker',
|
||||
['exec', database.container, 'psql', '-U', database.user, '-d', database.name, '-At', '-F', '\t', '-c', query],
|
||||
{ encoding: 'utf8' },
|
||||
).trim();
|
||||
}
|
||||
|
||||
function preflightMediaTools() {
|
||||
for (const command of ['ffprobe', 'ffmpeg']) {
|
||||
const result = spawnSync(command, ['-version'], { encoding: 'utf8' });
|
||||
assert(result.status === 0, `${command} is required for Keling video acceptance`);
|
||||
}
|
||||
}
|
||||
|
||||
function resolvePlatform() {
|
||||
const fields = runPSQL(`
|
||||
SELECT id::text, name, provider, status, base_url
|
||||
FROM integration_platforms
|
||||
WHERE platform_key = ${sqlLiteral(platformKey)}
|
||||
LIMIT 1`).split('\t');
|
||||
assert(fields.length === 5 && fields[0], `Platform ${platformKey} was not found`);
|
||||
assert(fields[2] === 'keling', `Platform ${platformKey} provider=${fields[2]}, expected keling`);
|
||||
assert(fields[3] === 'enabled', `Platform ${platformKey} status=${fields[3]}, expected enabled`);
|
||||
assert(
|
||||
fields[4].replace(/\/+$/, '') === 'https://api-aigv.transtreams.com/kling/v1',
|
||||
`Platform ${platformKey} base URL is not the configured TranStreams Kling endpoint`,
|
||||
);
|
||||
return { id: fields[0], name: fields[1], provider: fields[2], status: fields[3] };
|
||||
}
|
||||
|
||||
function validatePlatformModels(platformId) {
|
||||
const output = runPSQL(`
|
||||
SELECT provider_model_name, model_name, model_alias, enabled::text, model_type::text, capabilities::text
|
||||
FROM platform_models
|
||||
WHERE platform_id = ${sqlLiteral(platformId)}::uuid
|
||||
AND model_alias IN ('kling-o1', 'kling-3.0-omni')
|
||||
ORDER BY model_alias`);
|
||||
const rows = output
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((line) => line.split('\t'));
|
||||
const expectedModels = new Map([
|
||||
['kling-o1', 'kling-video-o1'],
|
||||
['kling-3.0-omni', 'kling-v3-omni'],
|
||||
]);
|
||||
for (const [gatewayModel, providerModel] of expectedModels) {
|
||||
const row = rows.find((candidate) => candidate[2] === gatewayModel);
|
||||
assert(row, `Platform ${platformKey} is missing gateway model alias ${gatewayModel}`);
|
||||
assert(row[0] === providerModel, `Platform model ${gatewayModel} provider_model_name=${row[0]}, expected ${providerModel}`);
|
||||
assert(row[1] === gatewayModel, `Platform model ${gatewayModel} model_name=${row[1]}, expected ${gatewayModel}`);
|
||||
assert(row[3] === 'true', `Platform model ${gatewayModel} is disabled`);
|
||||
const modelTypes = JSON.parse(row[4]);
|
||||
const capabilities = JSON.parse(row[5]);
|
||||
assert(modelTypes.includes('omni_video'), `Platform model ${gatewayModel} must include omni_video`);
|
||||
const omni = capabilities.omni_video || {};
|
||||
assert(Array.isArray(omni.output_resolutions), `Platform model ${gatewayModel} is missing omni_video.output_resolutions`);
|
||||
assert(omni.output_resolutions.includes('720p'), `Platform model ${gatewayModel} does not advertise 720p`);
|
||||
assert(omni.output_resolutions.includes('1080p'), `Platform model ${gatewayModel} does not advertise 1080p`);
|
||||
const expectedOutputAudio = gatewayModel !== 'kling-o1';
|
||||
assert(
|
||||
omni.output_audio === expectedOutputAudio,
|
||||
`Platform model ${gatewayModel} omni_video.output_audio=${omni.output_audio}, expected ${expectedOutputAudio}`,
|
||||
);
|
||||
}
|
||||
return rows.map((row) => ({ providerModelName: row[0], modelName: row[1], modelAlias: row[2], enabled: row[3] === 'true' }));
|
||||
}
|
||||
|
||||
function resolveGatewayAPIKey() {
|
||||
if (process.env.GATEWAY_E2E_API_KEY) {
|
||||
assert(process.env.GATEWAY_E2E_API_KEY_ID, 'Set GATEWAY_E2E_API_KEY_ID when GATEWAY_E2E_API_KEY is provided');
|
||||
return { ...apiKeyMetadata(process.env.GATEWAY_E2E_API_KEY_ID), secret: process.env.GATEWAY_E2E_API_KEY, source: 'environment' };
|
||||
}
|
||||
const fields = runPSQL(`
|
||||
SELECT key.id::text,
|
||||
key.key_secret,
|
||||
key.gateway_user_id::text,
|
||||
wallet.balance::float8,
|
||||
wallet.frozen_balance::float8
|
||||
FROM gateway_api_keys key
|
||||
JOIN gateway_wallet_accounts wallet
|
||||
ON wallet.gateway_user_id = key.gateway_user_id
|
||||
AND wallet.currency = 'resource'
|
||||
AND wallet.status = 'active'
|
||||
WHERE key.deleted_at IS NULL
|
||||
AND key.status = 'active'
|
||||
AND COALESCE(key.key_secret, '') <> ''
|
||||
AND (key.expires_at IS NULL OR key.expires_at > now())
|
||||
AND (key.scopes ? 'video' OR key.scopes ? '*' OR key.scopes ? 'all')
|
||||
AND (wallet.balance - wallet.frozen_balance) > 0
|
||||
ORDER BY (wallet.balance - wallet.frozen_balance) DESC, key.created_at DESC
|
||||
LIMIT 1`).split('\t');
|
||||
assert(fields.length === 5 && fields[0] && fields[1], 'No recoverable video-scoped Gateway API Key with positive balance was found');
|
||||
return {
|
||||
keyId: fields[0],
|
||||
secret: fields[1],
|
||||
userId: fields[2],
|
||||
balance: Number(fields[3]),
|
||||
frozenBalance: Number(fields[4]),
|
||||
source: 'database',
|
||||
};
|
||||
}
|
||||
|
||||
function apiKeyMetadata(keyId) {
|
||||
const fields = runPSQL(`
|
||||
SELECT key.id::text, key.gateway_user_id::text, wallet.balance::float8, wallet.frozen_balance::float8
|
||||
FROM gateway_api_keys key
|
||||
JOIN gateway_wallet_accounts wallet
|
||||
ON wallet.gateway_user_id = key.gateway_user_id
|
||||
AND wallet.currency = 'resource'
|
||||
WHERE key.id = ${sqlLiteral(keyId)}::uuid
|
||||
LIMIT 1`).split('\t');
|
||||
assert(fields.length === 4 && fields[0], `Gateway API Key ${keyId} was not found`);
|
||||
return { keyId: fields[0], userId: fields[1], balance: Number(fields[2]), frozenBalance: Number(fields[3]) };
|
||||
}
|
||||
|
||||
function walletState(userId) {
|
||||
const fields = runPSQL(`
|
||||
SELECT balance::float8, frozen_balance::float8
|
||||
FROM gateway_wallet_accounts
|
||||
WHERE gateway_user_id = ${sqlLiteral(userId)}::uuid
|
||||
AND currency = 'resource'
|
||||
LIMIT 1`).split('\t');
|
||||
assert(fields.length === 2, `Wallet for Gateway user ${userId} was not found`);
|
||||
return { balance: Number(fields[0]), frozenBalance: Number(fields[1]) };
|
||||
}
|
||||
|
||||
async function gatewayRequest(path, init = {}) {
|
||||
const response = await fetch(`${baseURL}${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
Authorization: `Bearer ${authentication.secret}`,
|
||||
...(init.body ? { 'Content-Type': 'application/json' } : {}),
|
||||
...(init.headers || {}),
|
||||
},
|
||||
});
|
||||
const text = await response.text();
|
||||
let body = {};
|
||||
try {
|
||||
body = text ? JSON.parse(text) : {};
|
||||
} catch {
|
||||
throw new Error(`${init.method || 'GET'} ${path} returned non-JSON HTTP ${response.status}`);
|
||||
}
|
||||
if (!response.ok) {
|
||||
const code = body.code ?? body.error?.code ?? 'unknown';
|
||||
const message = body.message ?? body.error?.message ?? 'request failed';
|
||||
throw new Error(`${init.method || 'GET'} ${path} failed HTTP ${response.status}, code=${code}, message=${message}`);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
async function createTask(validationCase) {
|
||||
let taskId = '';
|
||||
if (validationCase.interface === 'kling-compatible') {
|
||||
const accepted = await gatewayRequest('/v1/videos/omni-video', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(validationCase.request),
|
||||
});
|
||||
assert(accepted.code === 0, `${validationCase.name} compatibility create code=${accepted.code}`);
|
||||
assert(accepted.data?.task_status === 'submitted', `${validationCase.name} was not submitted`);
|
||||
taskId = accepted.data?.task_id;
|
||||
} else {
|
||||
const accepted = await gatewayRequest('/api/v1/videos/generations', {
|
||||
method: 'POST',
|
||||
headers: { 'X-Async': 'true' },
|
||||
body: JSON.stringify(validationCase.request),
|
||||
});
|
||||
taskId = accepted.taskId || accepted.task?.id;
|
||||
}
|
||||
assert(taskId, `${validationCase.name} create response is missing a Gateway task ID`);
|
||||
submittedTasks.push({ name: validationCase.name, taskId, model: validationCase.requestedModel, reusedExistingTask: false });
|
||||
return taskId;
|
||||
}
|
||||
|
||||
async function pollTask(validationCase, taskId) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < taskTimeoutMs) {
|
||||
if (validationCase.interface === 'kling-compatible') {
|
||||
const response = await gatewayRequest(`/v1/videos/omni-video/${taskId}`);
|
||||
const status = response.data?.task_status;
|
||||
if (status === 'succeed') break;
|
||||
if (status === 'failed') {
|
||||
throw new Error(`${validationCase.name} failed, code=${response.data?.task_status_code || 'unknown'}, message=${response.data?.task_status_msg || 'unknown'}`);
|
||||
}
|
||||
} else {
|
||||
const task = await gatewayRequest(`/api/v1/tasks/${taskId}`);
|
||||
if (task.status === 'succeeded') return task;
|
||||
if (task.status === 'failed' || task.status === 'cancelled') {
|
||||
throw new Error(`${validationCase.name} failed, code=${task.errorCode || 'unknown'}, message=${task.errorMessage || task.error || 'unknown'}`);
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
||||
}
|
||||
const task = await gatewayRequest(`/api/v1/tasks/${taskId}`);
|
||||
if (task.status !== 'succeeded') {
|
||||
throw new Error(`${validationCase.name} timed out after ${taskTimeoutMs}ms with status=${task.status}`);
|
||||
}
|
||||
return task;
|
||||
}
|
||||
|
||||
async function taskEvents(taskId) {
|
||||
const response = await fetch(`${baseURL}/api/v1/tasks/${taskId}/events`, {
|
||||
headers: { Authorization: `Bearer ${authentication.secret}` },
|
||||
});
|
||||
const text = await response.text();
|
||||
assert(response.ok, `GET task events failed HTTP ${response.status}`);
|
||||
return text
|
||||
.split(/\r?\n\r?\n/)
|
||||
.flatMap((block) => {
|
||||
const data = block
|
||||
.split(/\r?\n/)
|
||||
.filter((line) => line.startsWith('data:'))
|
||||
.map((line) => line.slice(5).trim())
|
||||
.join('\n');
|
||||
return data ? [JSON.parse(data)] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function taskPreprocessingLogs(taskId) {
|
||||
return gatewayRequest(`/api/v1/tasks/${taskId}/param-preprocessing`);
|
||||
}
|
||||
|
||||
function videoURLFromTask(task) {
|
||||
const data = Array.isArray(task.result?.data) ? task.result.data : [];
|
||||
return data.find((item) => item?.type === 'video')?.url || data.find((item) => item?.url)?.url || '';
|
||||
}
|
||||
|
||||
async function downloadVideo(url, filePath) {
|
||||
const response = await fetch(url);
|
||||
assert(response.ok, `Video download failed HTTP ${response.status}`);
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
assert(bytes.length > 0, 'Downloaded video is empty');
|
||||
await writeFile(filePath, bytes);
|
||||
return bytes.length;
|
||||
}
|
||||
|
||||
function probeVideo(filePath) {
|
||||
return JSON.parse(
|
||||
execFileSync('ffprobe', ['-v', 'error', '-show_streams', '-show_format', '-of', 'json', filePath], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function probeAudioVolume(filePath) {
|
||||
const result = spawnSync(
|
||||
'ffmpeg',
|
||||
['-hide_banner', '-nostats', '-i', filePath, '-map', '0:a:0', '-af', 'volumedetect', '-f', 'null', '-'],
|
||||
{ encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
assert(result.status === 0, 'ffmpeg volume analysis failed');
|
||||
const diagnostics = `${result.stdout || ''}\n${result.stderr || ''}`;
|
||||
const maxMatch = diagnostics.match(/max_volume:\s*(-?(?:\d+(?:\.\d+)?|inf))\s*dB/i);
|
||||
const meanMatch = diagnostics.match(/mean_volume:\s*(-?(?:\d+(?:\.\d+)?|inf))\s*dB/i);
|
||||
assert(maxMatch, 'ffmpeg did not report max_volume');
|
||||
const maxVolumeDb = Number(maxMatch[1]);
|
||||
assert(Number.isFinite(maxVolumeDb), 'Audio stream is fully silent (max_volume=-inf)');
|
||||
return { maxVolumeDb, meanVolumeDb: meanMatch && Number.isFinite(Number(meanMatch[1])) ? Number(meanMatch[1]) : null };
|
||||
}
|
||||
|
||||
function validateRequestSnapshots(task, validationCase) {
|
||||
const request = task.request || {};
|
||||
assert(request.model === validationCase.gatewayModel, `Task ${task.id} request.model=${request.model}, expected ${validationCase.gatewayModel}`);
|
||||
assert(request.resolution === validationCase.expected.resolution, `Task ${task.id} request.resolution=${request.resolution}`);
|
||||
assert(request.aspect_ratio === validationCase.request.aspect_ratio, `Task ${task.id} request.aspect_ratio=${request.aspect_ratio}`);
|
||||
assert(Number(request.duration) === validationCase.expected.duration, `Task ${task.id} request.duration=${request.duration}`);
|
||||
assert(request.audio === validationCase.expected.audio, `Task ${task.id} request.audio=${request.audio}`);
|
||||
if (validationCase.interface === 'kling-compatible') {
|
||||
assert(request._gateway_compatibility === 'keling_omni_v1', `Task ${task.id} compatibility marker is missing`);
|
||||
assert(request.external_task_id === validationCase.request.external_task_id, `Task ${task.id} external_task_id was not preserved`);
|
||||
}
|
||||
}
|
||||
|
||||
function validateTaskAudit(task, events, validationCase) {
|
||||
const attempts = Array.isArray(task.attempts) ? task.attempts : [];
|
||||
assert(attempts.length === 1, `Task ${task.id} has ${attempts.length} attempts, expected exactly one`);
|
||||
const attempt = attempts[0];
|
||||
assert(attempt.status === 'succeeded', `Task ${task.id} attempt status=${attempt.status}`);
|
||||
assert(attempt.platformName === platform.name, `Task ${task.id} used platform ${attempt.platformName}, expected ${platform.name}`);
|
||||
assert(attempt.provider === 'keling', `Task ${task.id} used provider ${attempt.provider}, expected keling`);
|
||||
assert(
|
||||
attempt.providerModelName === validationCase.providerModel,
|
||||
`Task ${task.id} providerModelName=${attempt.providerModelName}, expected ${validationCase.providerModel}`,
|
||||
);
|
||||
assert(task.remoteTaskId, `Task ${task.id} is missing remoteTaskId`);
|
||||
assert(attempt.requestId, `Task ${task.id} attempt is missing requestId`);
|
||||
const attemptRequest = attempt.requestSnapshot || {};
|
||||
assert(attemptRequest.model === validationCase.gatewayModel, `Task ${task.id} attempt request.model=${attemptRequest.model}`);
|
||||
assert(attemptRequest.resolution === validationCase.expected.resolution, `Task ${task.id} attempt request.resolution=${attemptRequest.resolution}`);
|
||||
assert(attemptRequest.aspect_ratio === validationCase.request.aspect_ratio, `Task ${task.id} attempt request.aspect_ratio=${attemptRequest.aspect_ratio}`);
|
||||
assert(Number(attemptRequest.duration) === validationCase.expected.duration, `Task ${task.id} attempt request.duration=${attemptRequest.duration}`);
|
||||
assert(attemptRequest.audio === validationCase.expected.audio, `Task ${task.id} attempt request.audio=${attemptRequest.audio}`);
|
||||
assert(Array.isArray(task.billings) && task.billings.length > 0, `Task ${task.id} has no billing lines`);
|
||||
assert(Number(task.finalChargeAmount) > 0, `Task ${task.id} finalChargeAmount must be positive`);
|
||||
assert(nearlyEqual(task.billingSummary?.totalAmount, task.finalChargeAmount), `Task ${task.id} billing total does not match final charge`);
|
||||
const startedEvents = events.filter((event) => event.eventType === 'task.attempt.started');
|
||||
assert(startedEvents.length === 1, `Task ${task.id} emitted ${startedEvents.length} task.attempt.started events`);
|
||||
return attempt;
|
||||
}
|
||||
|
||||
function inspectMedia(probe, filePath, taskId) {
|
||||
const streams = Array.isArray(probe.streams) ? probe.streams : [];
|
||||
const video = streams.find((stream) => stream.codec_type === 'video');
|
||||
const audioStreams = streams.filter((stream) => stream.codec_type === 'audio');
|
||||
assert(video, `Task ${taskId} output has no video stream`);
|
||||
const width = Number(video.width);
|
||||
const height = Number(video.height);
|
||||
const duration = Number(video.duration || probe.format?.duration);
|
||||
const shortEdge = Math.min(width, height);
|
||||
const ratio = width / height;
|
||||
assert(Number.isFinite(duration), `Task ${taskId} output duration is unavailable`);
|
||||
const volume = audioStreams.length > 0 ? probeAudioVolume(filePath) : null;
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
shortEdge,
|
||||
duration,
|
||||
ratio,
|
||||
videoCodec: video.codec_name || null,
|
||||
audioStreams: audioStreams.map((stream) => ({
|
||||
codec: stream.codec_name || null,
|
||||
channels: Number(stream.channels || 0),
|
||||
sampleRate: Number(stream.sample_rate || 0),
|
||||
})),
|
||||
volume,
|
||||
};
|
||||
}
|
||||
|
||||
function validateMedia(observed, expected, taskId) {
|
||||
const ratioError = Math.abs(observed.ratio - expected.ratio) / expected.ratio;
|
||||
observed.ratioErrorPercent = ratioError * 100;
|
||||
assert(observed.shortEdge >= expected.shortEdge, `Task ${taskId} short edge=${observed.shortEdge}, expected at least ${expected.shortEdge}`);
|
||||
assert(ratioError <= 0.01, `Task ${taskId} aspect ratio error=${(ratioError * 100).toFixed(2)}%, expected <=1%`);
|
||||
assert(Math.abs(observed.duration - expected.duration) <= 1, `Task ${taskId} duration=${observed.duration}s, expected ${expected.duration}s ±1s`);
|
||||
if (expected.audio) {
|
||||
assert(observed.audioStreams.length > 0, `Task ${taskId} requested audio but output has no audio stream`);
|
||||
assert(observed.volume && Number.isFinite(observed.volume.maxVolumeDb), `Task ${taskId} audio stream is fully silent`);
|
||||
} else {
|
||||
assert(observed.audioStreams.length === 0, `Task ${taskId} disabled audio but output has ${observed.audioStreams.length} audio stream(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizedFailureMessage(error) {
|
||||
return String(error instanceof Error ? error.message : error)
|
||||
.replace(/https?:\/\/[^\s,}]+/gi, '[redacted-url]')
|
||||
.replace(/Authorization:\s*Bearer\s+\S+/gi, 'Authorization: Bearer [redacted]');
|
||||
}
|
||||
|
||||
async function writeFailureReport(error) {
|
||||
const report = {
|
||||
ok: false,
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseURL,
|
||||
platform: { key: platformKey, name: platform?.name || null, provider: platform?.provider || null },
|
||||
authentication: authentication ? { type: 'gateway_api_key', source: authentication.source, keyId: authentication.keyId } : null,
|
||||
selectedCases: validationCases.map((item) => ({ name: item.name, interface: item.interface, model: item.requestedModel })),
|
||||
submittedTasks,
|
||||
error: sanitizedFailureMessage(error),
|
||||
};
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
return report;
|
||||
}
|
||||
|
||||
let platform;
|
||||
let authentication;
|
||||
|
||||
async function main() {
|
||||
preflightMediaTools();
|
||||
platform = resolvePlatform();
|
||||
const platformModels = validatePlatformModels(platform.id);
|
||||
authentication = resolveGatewayAPIKey();
|
||||
const health = await fetch(`${baseURL}/healthz`);
|
||||
assert(health.ok, `Gateway health check failed HTTP ${health.status}`);
|
||||
|
||||
const walletBefore = walletState(authentication.userId);
|
||||
const tempDirectory = await mkdtemp(join(tmpdir(), 'keling-video-e2e-'));
|
||||
const results = [];
|
||||
try {
|
||||
for (const validationCase of validationCases) {
|
||||
const resumedTaskId = String(resumeTaskIds[validationCase.name] || '').trim();
|
||||
let taskId;
|
||||
if (resumedTaskId) {
|
||||
taskId = resumedTaskId;
|
||||
submittedTasks.push({
|
||||
name: validationCase.name,
|
||||
taskId,
|
||||
model: validationCase.requestedModel,
|
||||
reusedExistingTask: true,
|
||||
});
|
||||
console.error(`Revalidating existing task for ${validationCase.name}`);
|
||||
} else {
|
||||
console.error(`Submitting ${validationCase.name} exactly once`);
|
||||
taskId = await createTask(validationCase);
|
||||
}
|
||||
const task = await pollTask(validationCase, taskId);
|
||||
const [events, preprocessing] = await Promise.all([taskEvents(taskId), taskPreprocessingLogs(taskId)]);
|
||||
validateRequestSnapshots(task, validationCase);
|
||||
const attempt = validateTaskAudit(task, events, validationCase);
|
||||
assert(Array.isArray(preprocessing.items), `Task ${taskId} preprocessing audit is unavailable`);
|
||||
const videoURL = videoURLFromTask(task);
|
||||
assert(videoURL, `Task ${taskId} result is missing a video URL`);
|
||||
const videoPath = join(tempDirectory, `${validationCase.name}.mp4`);
|
||||
const byteSize = await downloadVideo(videoURL, videoPath);
|
||||
const observed = inspectMedia(probeVideo(videoPath), videoPath, taskId);
|
||||
let validationError = '';
|
||||
try {
|
||||
validateMedia(observed, validationCase.expected, taskId);
|
||||
} catch (error) {
|
||||
validationError = sanitizedFailureMessage(error);
|
||||
if (!continueOnValidationFailure) throw error;
|
||||
}
|
||||
results.push({
|
||||
passed: validationError === '',
|
||||
validationError: validationError || undefined,
|
||||
name: validationCase.name,
|
||||
interface: validationCase.interface,
|
||||
taskId,
|
||||
remoteTaskId: task.remoteTaskId,
|
||||
requestId: attempt.requestId,
|
||||
platform: { key: platformKey, name: attempt.platformName, provider: attempt.provider },
|
||||
requestedModel: validationCase.requestedModel,
|
||||
providerModelName: attempt.providerModelName,
|
||||
reusedExistingTask: Boolean(resumedTaskId),
|
||||
requested: {
|
||||
resolution: validationCase.expected.resolution,
|
||||
aspectRatio: validationCase.request.aspect_ratio,
|
||||
duration: validationCase.expected.duration,
|
||||
audio: validationCase.expected.audio,
|
||||
},
|
||||
observed,
|
||||
byteSize,
|
||||
attemptCount: task.attemptCount,
|
||||
billingLineCount: task.billings.length,
|
||||
finalChargeAmount: Number(task.finalChargeAmount),
|
||||
billingSummary: task.billingSummary || {},
|
||||
eventTypes: events.map((event) => event.eventType),
|
||||
preprocessingChangeCount: preprocessing.items.reduce((sum, item) => sum + Number(item.changeCount || 0), 0),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await rm(tempDirectory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const walletAfter = walletState(authentication.userId);
|
||||
const totalHistoricalCharge = results.reduce((sum, result) => sum + result.finalChargeAmount, 0);
|
||||
const totalChargeThisRun = results
|
||||
.filter((result) => !result.reusedExistingTask)
|
||||
.reduce((sum, result) => sum + result.finalChargeAmount, 0);
|
||||
const walletDebit = walletBefore.balance - walletAfter.balance;
|
||||
assert(nearlyEqual(walletDebit, totalChargeThisRun), `Wallet debit=${walletDebit}, expected this-run charge=${totalChargeThisRun}`);
|
||||
assert(
|
||||
nearlyEqual(walletAfter.frozenBalance, walletBefore.frozenBalance),
|
||||
`Wallet frozen balance changed from ${walletBefore.frozenBalance} to ${walletAfter.frozenBalance}`,
|
||||
);
|
||||
|
||||
const report = {
|
||||
ok: results.every((result) => result.passed),
|
||||
generatedAt: new Date().toISOString(),
|
||||
baseURL,
|
||||
platform: { key: platformKey, name: platform.name, provider: platform.provider },
|
||||
platformModels,
|
||||
authentication: { type: 'gateway_api_key', source: authentication.source, keyId: authentication.keyId },
|
||||
wallet: {
|
||||
balanceBefore: walletBefore.balance,
|
||||
balanceAfter: walletAfter.balance,
|
||||
frozenBalanceBefore: walletBefore.frozenBalance,
|
||||
frozenBalanceAfter: walletAfter.frozenBalance,
|
||||
debit: walletDebit,
|
||||
totalChargeThisRun,
|
||||
totalHistoricalCharge,
|
||||
},
|
||||
results,
|
||||
};
|
||||
await mkdir(dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
|
||||
console.log(JSON.stringify({ ...report, outputPath }, null, 2));
|
||||
if (!report.ok) process.exitCode = 1;
|
||||
}
|
||||
|
||||
main().catch(async (error) => {
|
||||
const report = await writeFailureReport(error);
|
||||
console.error(report.error);
|
||||
console.error(`Failure report: ${outputPath}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user