forked from wangbo/easyai
398 lines
13 KiB
JavaScript
398 lines
13 KiB
JavaScript
import crypto from "node:crypto";
|
||
import fs from "node:fs";
|
||
import path from "node:path";
|
||
import process from "node:process";
|
||
import { execFileSync } from "node:child_process";
|
||
import { createRequire } from "node:module";
|
||
import { fileURLToPath } from "node:url";
|
||
|
||
const require = createRequire(import.meta.url);
|
||
const Ajv = require("ajv");
|
||
const directory = path.dirname(fileURLToPath(import.meta.url));
|
||
const repository = path.resolve(directory, "../..");
|
||
const defaultSourcePath = path.join(directory, "catalog.source.json");
|
||
const schemaPath = path.join(directory, "catalog.schema.json");
|
||
export const publicBaseUrl =
|
||
"https://git.51easyai.com/wangbo/easyai/raw/branch/main/model-catalog/official";
|
||
const args = process.argv.slice(2);
|
||
const option = (name) => {
|
||
const index = args.indexOf("--" + name);
|
||
return index < 0 ? "" : String(args[index + 1] || "");
|
||
};
|
||
const stableValue = (value) => {
|
||
if (Array.isArray(value)) return value.map(stableValue);
|
||
if (value && typeof value === "object") {
|
||
return Object.keys(value)
|
||
.sort()
|
||
.reduce((result, key) => {
|
||
result[key] = stableValue(value[key]);
|
||
return result;
|
||
}, {});
|
||
}
|
||
return value;
|
||
};
|
||
const checksum = (payload) =>
|
||
crypto
|
||
.createHash("sha256")
|
||
.update(JSON.stringify(stableValue(payload)))
|
||
.digest("hex");
|
||
const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
|
||
const serializeJson = (value) => `${JSON.stringify(value, null, 2)}\n`;
|
||
|
||
function canonicalizeSource(source) {
|
||
return {
|
||
...source,
|
||
baseModels: source.baseModels.map((model) => {
|
||
const current = model.capabilities || {};
|
||
const capabilityKeys = [
|
||
...model.modelTypes,
|
||
...Object.keys(current).filter(
|
||
(key) => !model.modelTypes.includes(key),
|
||
),
|
||
];
|
||
const capabilities = Object.fromEntries(
|
||
capabilityKeys.map((type) => {
|
||
const { type: _ignoredType, ...config } = current[type] || {};
|
||
return [type, { type, ...config }];
|
||
}),
|
||
);
|
||
return { ...model, capabilities };
|
||
}),
|
||
};
|
||
}
|
||
|
||
function assertOrderedRanges(value, pathParts = []) {
|
||
if (Array.isArray(value)) {
|
||
const field = pathParts.at(-1) || "";
|
||
if (
|
||
(field.endsWith("_range") || field === "thinkingBudgetTokensRange") &&
|
||
value.length === 2 &&
|
||
value.every((item) => typeof item === "number") &&
|
||
value[0] > value[1]
|
||
) {
|
||
throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
|
||
}
|
||
value.forEach((item, index) =>
|
||
assertOrderedRanges(item, [...pathParts, String(index)]),
|
||
);
|
||
return;
|
||
}
|
||
if (!value || typeof value !== "object") return;
|
||
if (
|
||
typeof value.min === "number" &&
|
||
typeof value.max === "number" &&
|
||
value.min > value.max
|
||
) {
|
||
throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
|
||
}
|
||
Object.entries(value).forEach(([key, item]) =>
|
||
assertOrderedRanges(item, [...pathParts, key]),
|
||
);
|
||
}
|
||
|
||
export function validateSource(file = defaultSourcePath) {
|
||
const sourceText = fs.readFileSync(file, "utf8");
|
||
const source = JSON.parse(sourceText);
|
||
const ajv = new Ajv({ allErrors: true, strict: false });
|
||
const validate = ajv.compile(readJson(schemaPath));
|
||
if (!validate(source)) {
|
||
throw new Error(
|
||
`官方目录格式无效: ${ajv.errorsText(validate.errors, { separator: "; " })}`,
|
||
);
|
||
}
|
||
const providerCodes = new Set(source.providers.map((item) => item.code));
|
||
if (providerCodes.size !== source.providers.length) {
|
||
throw new Error("官方目录包含重复厂商 code");
|
||
}
|
||
const keys = new Set();
|
||
const identities = new Set();
|
||
for (const model of source.baseModels) {
|
||
if (keys.has(model.catalogKey))
|
||
throw new Error(`重复 catalogKey: ${model.catalogKey}`);
|
||
keys.add(model.catalogKey);
|
||
const identity = `${model.alias}\u0000${model.defaultRealModelName}`;
|
||
if (identities.has(identity))
|
||
throw new Error(
|
||
`重复模型身份: ${model.alias} / ${model.defaultRealModelName}`,
|
||
);
|
||
identities.add(identity);
|
||
if (model.providerCode && !providerCodes.has(model.providerCode)) {
|
||
throw new Error(
|
||
`模型 ${model.catalogKey} 引用了不存在的厂商 ${model.providerCode}`,
|
||
);
|
||
}
|
||
const capabilityKeys = Object.keys(model.capabilities);
|
||
const missingCapabilities = model.modelTypes.filter(
|
||
(type) => !capabilityKeys.includes(type),
|
||
);
|
||
const extraCapabilities = capabilityKeys.filter(
|
||
(type) => !model.modelTypes.includes(type),
|
||
);
|
||
if (missingCapabilities.length || extraCapabilities.length) {
|
||
throw new Error(
|
||
`模型 ${model.catalogKey} 的 modelTypes 与 capabilities 不一致` +
|
||
`（缺少: ${missingCapabilities.join(", ") || "无"}；` +
|
||
`多余: ${extraCapabilities.join(", ") || "无"}）`,
|
||
);
|
||
}
|
||
for (const [type, capability] of Object.entries(model.capabilities)) {
|
||
if (capability.type !== type) {
|
||
throw new Error(
|
||
`模型 ${model.catalogKey} 的能力 ${type}.type 必须等于 ${type}`,
|
||
);
|
||
}
|
||
}
|
||
const textToSpeech = model.capabilities.text_to_speech;
|
||
if (textToSpeech) {
|
||
const voices = Array.isArray(textToSpeech.voices)
|
||
? textToSpeech.voices
|
||
: [];
|
||
const voiceIds = voices.map((voice) => voice.id);
|
||
if (new Set(voiceIds).size !== voiceIds.length) {
|
||
throw new Error(`模型 ${model.catalogKey} 包含重复音色 ID`);
|
||
}
|
||
if (
|
||
textToSpeech.default_voice_id &&
|
||
!voiceIds.includes(textToSpeech.default_voice_id)
|
||
) {
|
||
throw new Error(
|
||
`模型 ${model.catalogKey} 的默认音色不在 voices 中: ${textToSpeech.default_voice_id}`,
|
||
);
|
||
}
|
||
}
|
||
if (model.catalogKey.startsWith("base:seed:")) {
|
||
if (!model.seedKey)
|
||
throw new Error(`种子模型 ${model.catalogKey} 必须包含 seedKey`);
|
||
} else if (model.seedKey !== undefined) {
|
||
throw new Error(`运行时模型 ${model.catalogKey} 不应包含 seedKey`);
|
||
}
|
||
assertOrderedRanges(model.capabilities, [model.catalogKey, "capabilities"]);
|
||
}
|
||
const canonicalText = serializeJson(canonicalizeSource(source));
|
||
if (sourceText !== canonicalText) {
|
||
throw new Error("官方目录格式不规范，请运行 yarn format 后再提交");
|
||
}
|
||
return source;
|
||
}
|
||
|
||
export function buildRelease(
|
||
source,
|
||
{ version, note = "", previousVersion = null } = {},
|
||
) {
|
||
const publishedAt = new Date().toISOString();
|
||
const selectedVersion = version || publishedAt.replace(/[-:.]/g, "");
|
||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(selectedVersion))
|
||
throw new Error("目录版本格式无效");
|
||
const payload = {
|
||
schemaVersion: source.schemaVersion,
|
||
providers: source.providers,
|
||
baseModels: source.baseModels,
|
||
providerModelPresets: [],
|
||
};
|
||
return {
|
||
version: selectedVersion,
|
||
publishedAt,
|
||
generatedAt: publishedAt,
|
||
...payload,
|
||
checksum: checksum(payload),
|
||
count: source.baseModels.length,
|
||
ttlSeconds: 21600,
|
||
previousVersion,
|
||
sourceRepository: "https://git.51easyai.com/wangbo/easyai",
|
||
sourcePath: "model-catalog/official/catalog.source.json",
|
||
...(note ? { note } : {}),
|
||
};
|
||
}
|
||
|
||
export function assertNoMissingModels(
|
||
previous,
|
||
source,
|
||
{ allowRemovals = false, removalReason = "" } = {},
|
||
) {
|
||
const keys = new Set(source.baseModels.map((model) => model.catalogKey));
|
||
const removed = (previous?.baseModels || [])
|
||
.filter((model) => !keys.has(model.catalogKey))
|
||
.map((model) => ({ catalogKey: model.catalogKey, alias: model.alias }));
|
||
if (removed.length && (!allowRemovals || !removalReason.trim())) {
|
||
throw new Error(
|
||
`维护源遗漏已发布模型：${JSON.stringify(removed)}。下线请保留原身份并设置 status=offline；确需移除时须声明 --allow-removals --removal-reason。`,
|
||
);
|
||
}
|
||
return removed;
|
||
}
|
||
|
||
export function prepareArtifacts(targetDirectory = directory, options = {}) {
|
||
const source = validateSource(
|
||
path.join(targetDirectory, "catalog.source.json"),
|
||
);
|
||
const removed = assertNoMissingModels(options.previous, source, options);
|
||
const release = buildRelease(source, {
|
||
...options,
|
||
previousVersion: options.previous?.version || null,
|
||
});
|
||
if (removed.length)
|
||
release.removals = {
|
||
models: removed,
|
||
reason: options.removalReason.trim(),
|
||
};
|
||
const releasePath = path.join(
|
||
targetDirectory,
|
||
"releases",
|
||
release.version,
|
||
"catalog.json",
|
||
);
|
||
if (fs.existsSync(releasePath))
|
||
throw new Error("该版本快照已存在，请生成新版本，不能覆盖历史文件");
|
||
const pointer = {
|
||
schemaVersion: 1,
|
||
version: release.version,
|
||
checksum: release.checksum,
|
||
publishedAt: release.publishedAt,
|
||
catalogUrl: `${publicBaseUrl}/releases/${release.version}/catalog.json`,
|
||
};
|
||
fs.mkdirSync(path.dirname(releasePath), { recursive: true });
|
||
fs.writeFileSync(releasePath, serializeJson(release));
|
||
fs.writeFileSync(
|
||
path.join(targetDirectory, "latest.json"),
|
||
serializeJson(pointer),
|
||
);
|
||
return { pointer, release };
|
||
}
|
||
|
||
export function checkArtifacts(targetDirectory = directory) {
|
||
const source = validateSource(
|
||
path.join(targetDirectory, "catalog.source.json"),
|
||
);
|
||
const pointer = readJson(path.join(targetDirectory, "latest.json"));
|
||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
|
||
throw new Error("目录版本格式无效");
|
||
const release = readJson(
|
||
path.join(targetDirectory, "releases", pointer.version, "catalog.json"),
|
||
);
|
||
const payload = {
|
||
schemaVersion: source.schemaVersion,
|
||
providers: source.providers,
|
||
baseModels: source.baseModels,
|
||
providerModelPresets: [],
|
||
};
|
||
if (
|
||
pointer.version !== release.version ||
|
||
pointer.checksum !== release.checksum ||
|
||
release.checksum !== checksum(payload) ||
|
||
release.checksum !==
|
||
checksum({
|
||
schemaVersion: release.schemaVersion,
|
||
providers: release.providers,
|
||
baseModels: release.baseModels,
|
||
providerModelPresets: release.providerModelPresets || [],
|
||
})
|
||
) {
|
||
throw new Error(
|
||
"维护源、最新指针和版本快照不一致，请重新准备版本并一并提交",
|
||
);
|
||
}
|
||
if (
|
||
pointer.catalogUrl !==
|
||
`${publicBaseUrl}/releases/${release.version}/catalog.json`
|
||
)
|
||
throw new Error("目录快照必须指向 easyai 仓库公开地址");
|
||
return {
|
||
version: release.version,
|
||
checksum: release.checksum,
|
||
baseModelCount: release.baseModels.length,
|
||
catalogUrl: pointer.catalogUrl,
|
||
};
|
||
}
|
||
|
||
function previousCommittedRelease() {
|
||
const git = (file) =>
|
||
JSON.parse(
|
||
execFileSync("git", ["show", `HEAD:model-catalog/official/${file}`], {
|
||
cwd: repository,
|
||
encoding: "utf8",
|
||
stdio: ["ignore", "pipe", "pipe"],
|
||
maxBuffer: 4 * 1024 * 1024,
|
||
}),
|
||
);
|
||
let pointer;
|
||
try {
|
||
pointer = git("latest.json");
|
||
} catch (error) {
|
||
// A missing pointer is only allowed for this repository's first catalog commit.
|
||
try {
|
||
execFileSync(
|
||
"git",
|
||
["cat-file", "-e", "HEAD:model-catalog/official/latest.json"],
|
||
{ cwd: repository, stdio: "ignore" },
|
||
);
|
||
} catch {
|
||
return null;
|
||
}
|
||
throw error;
|
||
}
|
||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
|
||
throw new Error("已提交目录版本格式无效");
|
||
return git(`releases/${pointer.version}/catalog.json`);
|
||
}
|
||
|
||
function main() {
|
||
const command = args[0] || "validate";
|
||
if (command === "format") {
|
||
fs.writeFileSync(
|
||
defaultSourcePath,
|
||
serializeJson(canonicalizeSource(readJson(defaultSourcePath))),
|
||
);
|
||
validateSource();
|
||
console.log(JSON.stringify({ action: "formatted" }));
|
||
} else if (command === "validate") {
|
||
const source = validateSource();
|
||
console.log(
|
||
JSON.stringify({
|
||
action: "validated",
|
||
baseModelCount: source.baseModels.length,
|
||
providerCount: source.providers.length,
|
||
}),
|
||
);
|
||
} else if (command === "prepare") {
|
||
const result = prepareArtifacts(directory, {
|
||
previous: previousCommittedRelease(),
|
||
version: option("version") || undefined,
|
||
note: option("note"),
|
||
allowRemovals: args.includes("--allow-removals"),
|
||
removalReason: option("removal-reason"),
|
||
});
|
||
console.log(
|
||
JSON.stringify(
|
||
{
|
||
action: "prepared-for-git-commit",
|
||
version: result.pointer.version,
|
||
checksum: result.pointer.checksum,
|
||
baseModelCount: result.release.count,
|
||
instruction:
|
||
"检查 Git 差异后，将维护源、latest.json 和版本快照一并提交并推送；公开 JSON 只来自 Git 已提交内容。",
|
||
},
|
||
null,
|
||
2,
|
||
),
|
||
);
|
||
} else if (command === "check") {
|
||
console.log(JSON.stringify(checkArtifacts(), null, 2));
|
||
} else {
|
||
throw new Error(
|
||
"支持 format、validate、prepare、check；目录通过 Git 提交发布，不再使用 OSS/CDN 上传命令。",
|
||
);
|
||
}
|
||
}
|
||
|
||
if (
|
||
process.argv[1] &&
|
||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||
) {
|
||
try {
|
||
main();
|
||
} catch (error) {
|
||
console.error(error.message);
|
||
process.exitCode = 1;
|
||
}
|
||
}
|