import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import process from "node:process"; import { execFileSync } from "node:child_process"; import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; const require = createRequire(import.meta.url); const Ajv = require("ajv"); const directory = path.dirname(fileURLToPath(import.meta.url)); const repository = path.resolve(directory, "../.."); const defaultSourcePath = path.join(directory, "catalog.source.json"); const schemaPath = path.join(directory, "catalog.schema.json"); export const publicBaseUrl = "https://git.51easyai.com/wangbo/easyai/raw/branch/main/model-catalog/official"; const args = process.argv.slice(2); const option = (name) => { const index = args.indexOf("--" + name); return index < 0 ? "" : String(args[index + 1] || ""); }; const stableValue = (value) => { if (Array.isArray(value)) return value.map(stableValue); if (value && typeof value === "object") { return Object.keys(value) .sort() .reduce((result, key) => { result[key] = stableValue(value[key]); return result; }, {}); } return value; }; const checksum = (payload) => crypto .createHash("sha256") .update(JSON.stringify(stableValue(payload))) .digest("hex"); const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8")); const serializeJson = (value) => `${JSON.stringify(value, null, 2)}\n`; function canonicalizeSource(source) { return { ...source, baseModels: source.baseModels.map((model) => { const current = model.capabilities || {}; const capabilityKeys = [ ...model.modelTypes, ...Object.keys(current).filter( (key) => !model.modelTypes.includes(key), ), ]; const capabilities = Object.fromEntries( capabilityKeys.map((type) => { const { type: _ignoredType, ...config } = current[type] || {}; return [type, { type, ...config }]; }), ); return { ...model, capabilities }; }), }; } function assertOrderedRanges(value, pathParts = []) { if (Array.isArray(value)) { const field = pathParts.at(-1) || ""; if ( (field.endsWith("_range") || field === "thinkingBudgetTokensRange") && value.length === 2 && value.every((item) => typeof item === "number") && value[0] > value[1] ) { throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`); } value.forEach((item, index) => assertOrderedRanges(item, [...pathParts, String(index)]), ); return; } if (!value || typeof value !== "object") return; if ( typeof value.min === "number" && typeof value.max === "number" && value.min > value.max ) { throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`); } Object.entries(value).forEach(([key, item]) => assertOrderedRanges(item, [...pathParts, key]), ); } export function validateSource(file = defaultSourcePath) { const sourceText = fs.readFileSync(file, "utf8"); const source = JSON.parse(sourceText); const ajv = new Ajv({ allErrors: true, strict: false }); const validate = ajv.compile(readJson(schemaPath)); if (!validate(source)) { throw new Error( `官方目录格式无效: ${ajv.errorsText(validate.errors, { separator: "; " })}`, ); } const providerCodes = new Set(source.providers.map((item) => item.code)); if (providerCodes.size !== source.providers.length) { throw new Error("官方目录包含重复厂商 code"); } const keys = new Set(); const identities = new Set(); for (const model of source.baseModels) { if (keys.has(model.catalogKey)) throw new Error(`重复 catalogKey: ${model.catalogKey}`); keys.add(model.catalogKey); const identity = `${model.alias}\u0000${model.defaultRealModelName}`; if (identities.has(identity)) throw new Error( `重复模型身份: ${model.alias} / ${model.defaultRealModelName}`, ); identities.add(identity); if (model.providerCode && !providerCodes.has(model.providerCode)) { throw new Error( `模型 ${model.catalogKey} 引用了不存在的厂商 ${model.providerCode}`, ); } const capabilityKeys = Object.keys(model.capabilities); const missingCapabilities = model.modelTypes.filter( (type) => !capabilityKeys.includes(type), ); const extraCapabilities = capabilityKeys.filter( (type) => !model.modelTypes.includes(type), ); if (missingCapabilities.length || extraCapabilities.length) { throw new Error( `模型 ${model.catalogKey} 的 modelTypes 与 capabilities 不一致` + `(缺少: ${missingCapabilities.join(", ") || "无"};` + `多余: ${extraCapabilities.join(", ") || "无"})`, ); } for (const [type, capability] of Object.entries(model.capabilities)) { if (capability.type !== type) { throw new Error( `模型 ${model.catalogKey} 的能力 ${type}.type 必须等于 ${type}`, ); } } const textToSpeech = model.capabilities.text_to_speech; if (textToSpeech) { const voices = Array.isArray(textToSpeech.voices) ? textToSpeech.voices : []; const voiceIds = voices.map((voice) => voice.id); if (new Set(voiceIds).size !== voiceIds.length) { throw new Error(`模型 ${model.catalogKey} 包含重复音色 ID`); } if ( textToSpeech.default_voice_id && !voiceIds.includes(textToSpeech.default_voice_id) ) { throw new Error( `模型 ${model.catalogKey} 的默认音色不在 voices 中: ${textToSpeech.default_voice_id}`, ); } } if (model.catalogKey.startsWith("base:seed:")) { if (!model.seedKey) throw new Error(`种子模型 ${model.catalogKey} 必须包含 seedKey`); } else if (model.seedKey !== undefined) { throw new Error(`运行时模型 ${model.catalogKey} 不应包含 seedKey`); } assertOrderedRanges(model.capabilities, [model.catalogKey, "capabilities"]); } const canonicalText = serializeJson(canonicalizeSource(source)); if (sourceText !== canonicalText) { throw new Error("官方目录格式不规范,请运行 yarn format 后再提交"); } return source; } export function buildRelease( source, { version, note = "", previousVersion = null } = {}, ) { const publishedAt = new Date().toISOString(); const selectedVersion = version || publishedAt.replace(/[-:.]/g, ""); if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(selectedVersion)) throw new Error("目录版本格式无效"); const payload = { schemaVersion: source.schemaVersion, providers: source.providers, baseModels: source.baseModels, providerModelPresets: [], }; return { version: selectedVersion, publishedAt, generatedAt: publishedAt, ...payload, checksum: checksum(payload), count: source.baseModels.length, ttlSeconds: 21600, previousVersion, sourceRepository: "https://git.51easyai.com/wangbo/easyai", sourcePath: "model-catalog/official/catalog.source.json", ...(note ? { note } : {}), }; } export function assertNoMissingModels( previous, source, { allowRemovals = false, removalReason = "" } = {}, ) { const keys = new Set(source.baseModels.map((model) => model.catalogKey)); const removed = (previous?.baseModels || []) .filter((model) => !keys.has(model.catalogKey)) .map((model) => ({ catalogKey: model.catalogKey, alias: model.alias })); if (removed.length && (!allowRemovals || !removalReason.trim())) { throw new Error( `维护源遗漏已发布模型:${JSON.stringify(removed)}。下线请保留原身份并设置 status=offline;确需移除时须声明 --allow-removals --removal-reason。`, ); } return removed; } export function prepareArtifacts(targetDirectory = directory, options = {}) { const source = validateSource( path.join(targetDirectory, "catalog.source.json"), ); const removed = assertNoMissingModels(options.previous, source, options); const release = buildRelease(source, { ...options, previousVersion: options.previous?.version || null, }); if (removed.length) release.removals = { models: removed, reason: options.removalReason.trim(), }; const releasePath = path.join( targetDirectory, "releases", release.version, "catalog.json", ); if (fs.existsSync(releasePath)) throw new Error("该版本快照已存在,请生成新版本,不能覆盖历史文件"); const pointer = { schemaVersion: 1, version: release.version, checksum: release.checksum, publishedAt: release.publishedAt, catalogUrl: `${publicBaseUrl}/releases/${release.version}/catalog.json`, }; fs.mkdirSync(path.dirname(releasePath), { recursive: true }); fs.writeFileSync(releasePath, serializeJson(release)); fs.writeFileSync( path.join(targetDirectory, "latest.json"), serializeJson(pointer), ); return { pointer, release }; } export function checkArtifacts(targetDirectory = directory) { const source = validateSource( path.join(targetDirectory, "catalog.source.json"), ); const pointer = readJson(path.join(targetDirectory, "latest.json")); if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version)) throw new Error("目录版本格式无效"); const release = readJson( path.join(targetDirectory, "releases", pointer.version, "catalog.json"), ); const payload = { schemaVersion: source.schemaVersion, providers: source.providers, baseModels: source.baseModels, providerModelPresets: [], }; if ( pointer.version !== release.version || pointer.checksum !== release.checksum || release.checksum !== checksum(payload) || release.checksum !== checksum({ schemaVersion: release.schemaVersion, providers: release.providers, baseModels: release.baseModels, providerModelPresets: release.providerModelPresets || [], }) ) { throw new Error( "维护源、最新指针和版本快照不一致,请重新准备版本并一并提交", ); } if ( pointer.catalogUrl !== `${publicBaseUrl}/releases/${release.version}/catalog.json` ) throw new Error("目录快照必须指向 easyai 仓库公开地址"); return { version: release.version, checksum: release.checksum, baseModelCount: release.baseModels.length, catalogUrl: pointer.catalogUrl, }; } function previousCommittedRelease() { const git = (file) => JSON.parse( execFileSync("git", ["show", `HEAD:model-catalog/official/${file}`], { cwd: repository, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 4 * 1024 * 1024, }), ); let pointer; try { pointer = git("latest.json"); } catch (error) { // A missing pointer is only allowed for this repository's first catalog commit. try { execFileSync( "git", ["cat-file", "-e", "HEAD:model-catalog/official/latest.json"], { cwd: repository, stdio: "ignore" }, ); } catch { return null; } throw error; } if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version)) throw new Error("已提交目录版本格式无效"); return git(`releases/${pointer.version}/catalog.json`); } function main() { const command = args[0] || "validate"; if (command === "format") { fs.writeFileSync( defaultSourcePath, serializeJson(canonicalizeSource(readJson(defaultSourcePath))), ); validateSource(); console.log(JSON.stringify({ action: "formatted" })); } else if (command === "validate") { const source = validateSource(); console.log( JSON.stringify({ action: "validated", baseModelCount: source.baseModels.length, providerCount: source.providers.length, }), ); } else if (command === "prepare") { const result = prepareArtifacts(directory, { previous: previousCommittedRelease(), version: option("version") || undefined, note: option("note"), allowRemovals: args.includes("--allow-removals"), removalReason: option("removal-reason"), }); console.log( JSON.stringify( { action: "prepared-for-git-commit", version: result.pointer.version, checksum: result.pointer.checksum, baseModelCount: result.release.count, instruction: "检查 Git 差异后,将维护源、latest.json 和版本快照一并提交并推送;公开 JSON 只来自 Git 已提交内容。", }, null, 2, ), ); } else if (command === "check") { console.log(JSON.stringify(checkArtifacts(), null, 2)); } else { throw new Error( "支持 format、validate、prepare、check;目录通过 Git 提交发布,不再使用 OSS/CDN 上传命令。", ); } } if ( process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) ) { try { main(); } catch (error) { console.error(error.message); process.exitCode = 1; } }