forked from wangbo/easyai
feat(catalog): 将官方模型目录迁入公开Git仓库
This commit is contained in:
@@ -0,0 +1,397 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const Ajv = require("ajv");
|
||||
const directory = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repository = path.resolve(directory, "../..");
|
||||
const defaultSourcePath = path.join(directory, "catalog.source.json");
|
||||
const schemaPath = path.join(directory, "catalog.schema.json");
|
||||
export const publicBaseUrl =
|
||||
"https://git.51easyai.com/wangbo/easyai/raw/branch/main/model-catalog/official";
|
||||
const args = process.argv.slice(2);
|
||||
const option = (name) => {
|
||||
const index = args.indexOf("--" + name);
|
||||
return index < 0 ? "" : String(args[index + 1] || "");
|
||||
};
|
||||
const stableValue = (value) => {
|
||||
if (Array.isArray(value)) return value.map(stableValue);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.keys(value)
|
||||
.sort()
|
||||
.reduce((result, key) => {
|
||||
result[key] = stableValue(value[key]);
|
||||
return result;
|
||||
}, {});
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const checksum = (payload) =>
|
||||
crypto
|
||||
.createHash("sha256")
|
||||
.update(JSON.stringify(stableValue(payload)))
|
||||
.digest("hex");
|
||||
const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
const serializeJson = (value) => `${JSON.stringify(value, null, 2)}\n`;
|
||||
|
||||
function canonicalizeSource(source) {
|
||||
return {
|
||||
...source,
|
||||
baseModels: source.baseModels.map((model) => {
|
||||
const current = model.capabilities || {};
|
||||
const capabilityKeys = [
|
||||
...model.modelTypes,
|
||||
...Object.keys(current).filter(
|
||||
(key) => !model.modelTypes.includes(key),
|
||||
),
|
||||
];
|
||||
const capabilities = Object.fromEntries(
|
||||
capabilityKeys.map((type) => {
|
||||
const { type: _ignoredType, ...config } = current[type] || {};
|
||||
return [type, { type, ...config }];
|
||||
}),
|
||||
);
|
||||
return { ...model, capabilities };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function assertOrderedRanges(value, pathParts = []) {
|
||||
if (Array.isArray(value)) {
|
||||
const field = pathParts.at(-1) || "";
|
||||
if (
|
||||
(field.endsWith("_range") || field === "thinkingBudgetTokensRange") &&
|
||||
value.length === 2 &&
|
||||
value.every((item) => typeof item === "number") &&
|
||||
value[0] > value[1]
|
||||
) {
|
||||
throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
|
||||
}
|
||||
value.forEach((item, index) =>
|
||||
assertOrderedRanges(item, [...pathParts, String(index)]),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== "object") return;
|
||||
if (
|
||||
typeof value.min === "number" &&
|
||||
typeof value.max === "number" &&
|
||||
value.min > value.max
|
||||
) {
|
||||
throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
|
||||
}
|
||||
Object.entries(value).forEach(([key, item]) =>
|
||||
assertOrderedRanges(item, [...pathParts, key]),
|
||||
);
|
||||
}
|
||||
|
||||
export function validateSource(file = defaultSourcePath) {
|
||||
const sourceText = fs.readFileSync(file, "utf8");
|
||||
const source = JSON.parse(sourceText);
|
||||
const ajv = new Ajv({ allErrors: true, strict: false });
|
||||
const validate = ajv.compile(readJson(schemaPath));
|
||||
if (!validate(source)) {
|
||||
throw new Error(
|
||||
`官方目录格式无效: ${ajv.errorsText(validate.errors, { separator: "; " })}`,
|
||||
);
|
||||
}
|
||||
const providerCodes = new Set(source.providers.map((item) => item.code));
|
||||
if (providerCodes.size !== source.providers.length) {
|
||||
throw new Error("官方目录包含重复厂商 code");
|
||||
}
|
||||
const keys = new Set();
|
||||
const identities = new Set();
|
||||
for (const model of source.baseModels) {
|
||||
if (keys.has(model.catalogKey))
|
||||
throw new Error(`重复 catalogKey: ${model.catalogKey}`);
|
||||
keys.add(model.catalogKey);
|
||||
const identity = `${model.alias}\u0000${model.defaultRealModelName}`;
|
||||
if (identities.has(identity))
|
||||
throw new Error(
|
||||
`重复模型身份: ${model.alias} / ${model.defaultRealModelName}`,
|
||||
);
|
||||
identities.add(identity);
|
||||
if (model.providerCode && !providerCodes.has(model.providerCode)) {
|
||||
throw new Error(
|
||||
`模型 ${model.catalogKey} 引用了不存在的厂商 ${model.providerCode}`,
|
||||
);
|
||||
}
|
||||
const capabilityKeys = Object.keys(model.capabilities);
|
||||
const missingCapabilities = model.modelTypes.filter(
|
||||
(type) => !capabilityKeys.includes(type),
|
||||
);
|
||||
const extraCapabilities = capabilityKeys.filter(
|
||||
(type) => !model.modelTypes.includes(type),
|
||||
);
|
||||
if (missingCapabilities.length || extraCapabilities.length) {
|
||||
throw new Error(
|
||||
`模型 ${model.catalogKey} 的 modelTypes 与 capabilities 不一致` +
|
||||
`（缺少: ${missingCapabilities.join(", ") || "无"}；` +
|
||||
`多余: ${extraCapabilities.join(", ") || "无"}）`,
|
||||
);
|
||||
}
|
||||
for (const [type, capability] of Object.entries(model.capabilities)) {
|
||||
if (capability.type !== type) {
|
||||
throw new Error(
|
||||
`模型 ${model.catalogKey} 的能力 ${type}.type 必须等于 ${type}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const textToSpeech = model.capabilities.text_to_speech;
|
||||
if (textToSpeech) {
|
||||
const voices = Array.isArray(textToSpeech.voices)
|
||||
? textToSpeech.voices
|
||||
: [];
|
||||
const voiceIds = voices.map((voice) => voice.id);
|
||||
if (new Set(voiceIds).size !== voiceIds.length) {
|
||||
throw new Error(`模型 ${model.catalogKey} 包含重复音色 ID`);
|
||||
}
|
||||
if (
|
||||
textToSpeech.default_voice_id &&
|
||||
!voiceIds.includes(textToSpeech.default_voice_id)
|
||||
) {
|
||||
throw new Error(
|
||||
`模型 ${model.catalogKey} 的默认音色不在 voices 中: ${textToSpeech.default_voice_id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (model.catalogKey.startsWith("base:seed:")) {
|
||||
if (!model.seedKey)
|
||||
throw new Error(`种子模型 ${model.catalogKey} 必须包含 seedKey`);
|
||||
} else if (model.seedKey !== undefined) {
|
||||
throw new Error(`运行时模型 ${model.catalogKey} 不应包含 seedKey`);
|
||||
}
|
||||
assertOrderedRanges(model.capabilities, [model.catalogKey, "capabilities"]);
|
||||
}
|
||||
const canonicalText = serializeJson(canonicalizeSource(source));
|
||||
if (sourceText !== canonicalText) {
|
||||
throw new Error("官方目录格式不规范，请运行 yarn format 后再提交");
|
||||
}
|
||||
return source;
|
||||
}
|
||||
|
||||
export function buildRelease(
|
||||
source,
|
||||
{ version, note = "", previousVersion = null } = {},
|
||||
) {
|
||||
const publishedAt = new Date().toISOString();
|
||||
const selectedVersion = version || publishedAt.replace(/[-:.]/g, "");
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(selectedVersion))
|
||||
throw new Error("目录版本格式无效");
|
||||
const payload = {
|
||||
schemaVersion: source.schemaVersion,
|
||||
providers: source.providers,
|
||||
baseModels: source.baseModels,
|
||||
providerModelPresets: [],
|
||||
};
|
||||
return {
|
||||
version: selectedVersion,
|
||||
publishedAt,
|
||||
generatedAt: publishedAt,
|
||||
...payload,
|
||||
checksum: checksum(payload),
|
||||
count: source.baseModels.length,
|
||||
ttlSeconds: 21600,
|
||||
previousVersion,
|
||||
sourceRepository: "https://git.51easyai.com/wangbo/easyai",
|
||||
sourcePath: "model-catalog/official/catalog.source.json",
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function assertNoMissingModels(
|
||||
previous,
|
||||
source,
|
||||
{ allowRemovals = false, removalReason = "" } = {},
|
||||
) {
|
||||
const keys = new Set(source.baseModels.map((model) => model.catalogKey));
|
||||
const removed = (previous?.baseModels || [])
|
||||
.filter((model) => !keys.has(model.catalogKey))
|
||||
.map((model) => ({ catalogKey: model.catalogKey, alias: model.alias }));
|
||||
if (removed.length && (!allowRemovals || !removalReason.trim())) {
|
||||
throw new Error(
|
||||
`维护源遗漏已发布模型：${JSON.stringify(removed)}。下线请保留原身份并设置 status=offline；确需移除时须声明 --allow-removals --removal-reason。`,
|
||||
);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function prepareArtifacts(targetDirectory = directory, options = {}) {
|
||||
const source = validateSource(
|
||||
path.join(targetDirectory, "catalog.source.json"),
|
||||
);
|
||||
const removed = assertNoMissingModels(options.previous, source, options);
|
||||
const release = buildRelease(source, {
|
||||
...options,
|
||||
previousVersion: options.previous?.version || null,
|
||||
});
|
||||
if (removed.length)
|
||||
release.removals = {
|
||||
models: removed,
|
||||
reason: options.removalReason.trim(),
|
||||
};
|
||||
const releasePath = path.join(
|
||||
targetDirectory,
|
||||
"releases",
|
||||
release.version,
|
||||
"catalog.json",
|
||||
);
|
||||
if (fs.existsSync(releasePath))
|
||||
throw new Error("该版本快照已存在，请生成新版本，不能覆盖历史文件");
|
||||
const pointer = {
|
||||
schemaVersion: 1,
|
||||
version: release.version,
|
||||
checksum: release.checksum,
|
||||
publishedAt: release.publishedAt,
|
||||
catalogUrl: `${publicBaseUrl}/releases/${release.version}/catalog.json`,
|
||||
};
|
||||
fs.mkdirSync(path.dirname(releasePath), { recursive: true });
|
||||
fs.writeFileSync(releasePath, serializeJson(release));
|
||||
fs.writeFileSync(
|
||||
path.join(targetDirectory, "latest.json"),
|
||||
serializeJson(pointer),
|
||||
);
|
||||
return { pointer, release };
|
||||
}
|
||||
|
||||
export function checkArtifacts(targetDirectory = directory) {
|
||||
const source = validateSource(
|
||||
path.join(targetDirectory, "catalog.source.json"),
|
||||
);
|
||||
const pointer = readJson(path.join(targetDirectory, "latest.json"));
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
|
||||
throw new Error("目录版本格式无效");
|
||||
const release = readJson(
|
||||
path.join(targetDirectory, "releases", pointer.version, "catalog.json"),
|
||||
);
|
||||
const payload = {
|
||||
schemaVersion: source.schemaVersion,
|
||||
providers: source.providers,
|
||||
baseModels: source.baseModels,
|
||||
providerModelPresets: [],
|
||||
};
|
||||
if (
|
||||
pointer.version !== release.version ||
|
||||
pointer.checksum !== release.checksum ||
|
||||
release.checksum !== checksum(payload) ||
|
||||
release.checksum !==
|
||||
checksum({
|
||||
schemaVersion: release.schemaVersion,
|
||||
providers: release.providers,
|
||||
baseModels: release.baseModels,
|
||||
providerModelPresets: release.providerModelPresets || [],
|
||||
})
|
||||
) {
|
||||
throw new Error(
|
||||
"维护源、最新指针和版本快照不一致，请重新准备版本并一并提交",
|
||||
);
|
||||
}
|
||||
if (
|
||||
pointer.catalogUrl !==
|
||||
`${publicBaseUrl}/releases/${release.version}/catalog.json`
|
||||
)
|
||||
throw new Error("目录快照必须指向 easyai 仓库公开地址");
|
||||
return {
|
||||
version: release.version,
|
||||
checksum: release.checksum,
|
||||
baseModelCount: release.baseModels.length,
|
||||
catalogUrl: pointer.catalogUrl,
|
||||
};
|
||||
}
|
||||
|
||||
function previousCommittedRelease() {
|
||||
const git = (file) =>
|
||||
JSON.parse(
|
||||
execFileSync("git", ["show", `HEAD:model-catalog/official/${file}`], {
|
||||
cwd: repository,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
}),
|
||||
);
|
||||
let pointer;
|
||||
try {
|
||||
pointer = git("latest.json");
|
||||
} catch (error) {
|
||||
// A missing pointer is only allowed for this repository's first catalog commit.
|
||||
try {
|
||||
execFileSync(
|
||||
"git",
|
||||
["cat-file", "-e", "HEAD:model-catalog/official/latest.json"],
|
||||
{ cwd: repository, stdio: "ignore" },
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
|
||||
throw new Error("已提交目录版本格式无效");
|
||||
return git(`releases/${pointer.version}/catalog.json`);
|
||||
}
|
||||
|
||||
function main() {
|
||||
const command = args[0] || "validate";
|
||||
if (command === "format") {
|
||||
fs.writeFileSync(
|
||||
defaultSourcePath,
|
||||
serializeJson(canonicalizeSource(readJson(defaultSourcePath))),
|
||||
);
|
||||
validateSource();
|
||||
console.log(JSON.stringify({ action: "formatted" }));
|
||||
} else if (command === "validate") {
|
||||
const source = validateSource();
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
action: "validated",
|
||||
baseModelCount: source.baseModels.length,
|
||||
providerCount: source.providers.length,
|
||||
}),
|
||||
);
|
||||
} else if (command === "prepare") {
|
||||
const result = prepareArtifacts(directory, {
|
||||
previous: previousCommittedRelease(),
|
||||
version: option("version") || undefined,
|
||||
note: option("note"),
|
||||
allowRemovals: args.includes("--allow-removals"),
|
||||
removalReason: option("removal-reason"),
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
action: "prepared-for-git-commit",
|
||||
version: result.pointer.version,
|
||||
checksum: result.pointer.checksum,
|
||||
baseModelCount: result.release.count,
|
||||
instruction:
|
||||
"检查 Git 差异后，将维护源、latest.json 和版本快照一并提交并推送；公开 JSON 只来自 Git 已提交内容。",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} else if (command === "check") {
|
||||
console.log(JSON.stringify(checkArtifacts(), null, 2));
|
||||
} else {
|
||||
throw new Error(
|
||||
"支持 format、validate、prepare、check；目录通过 Git 提交发布，不再使用 OSS/CDN 上传命令。",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user