feat(catalog): 将官方模型目录迁入公开Git仓库

model-catalog/official/catalog.mjs

@@ -0,0 +1,397 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { execFileSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { fileURLToPath } from "node:url";
+
+const require = createRequire(import.meta.url);
+const Ajv = require("ajv");
+const directory = path.dirname(fileURLToPath(import.meta.url));
+const repository = path.resolve(directory, "../..");
+const defaultSourcePath = path.join(directory, "catalog.source.json");
+const schemaPath = path.join(directory, "catalog.schema.json");
+export const publicBaseUrl =
+  "https://git.51easyai.com/wangbo/easyai/raw/branch/main/model-catalog/official";
+const args = process.argv.slice(2);
+const option = (name) => {
+  const index = args.indexOf("--" + name);
+  return index < 0 ? "" : String(args[index + 1] || "");
+};
+const stableValue = (value) => {
+  if (Array.isArray(value)) return value.map(stableValue);
+  if (value && typeof value === "object") {
+    return Object.keys(value)
+      .sort()
+      .reduce((result, key) => {
+        result[key] = stableValue(value[key]);
+        return result;
+      }, {});
+  }
+  return value;
+};
+const checksum = (payload) =>
+  crypto
+    .createHash("sha256")
+    .update(JSON.stringify(stableValue(payload)))
+    .digest("hex");
+const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
+const serializeJson = (value) => `${JSON.stringify(value, null, 2)}\n`;
+
+function canonicalizeSource(source) {
+  return {
+    ...source,
+    baseModels: source.baseModels.map((model) => {
+      const current = model.capabilities || {};
+      const capabilityKeys = [
+        ...model.modelTypes,
+        ...Object.keys(current).filter(
+          (key) => !model.modelTypes.includes(key),
+        ),
+      ];
+      const capabilities = Object.fromEntries(
+        capabilityKeys.map((type) => {
+          const { type: _ignoredType, ...config } = current[type] || {};
+          return [type, { type, ...config }];
+        }),
+      );
+      return { ...model, capabilities };
+    }),
+  };
+}
+
+function assertOrderedRanges(value, pathParts = []) {
+  if (Array.isArray(value)) {
+    const field = pathParts.at(-1) || "";
+    if (
+      (field.endsWith("_range") || field === "thinkingBudgetTokensRange") &&
+      value.length === 2 &&
+      value.every((item) => typeof item === "number") &&
+      value[0] > value[1]
+    ) {
+      throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
+    }
+    value.forEach((item, index) =>
+      assertOrderedRanges(item, [...pathParts, String(index)]),
+    );
+    return;
+  }
+  if (!value || typeof value !== "object") return;
+  if (
+    typeof value.min === "number" &&
+    typeof value.max === "number" &&
+    value.min > value.max
+  ) {
+    throw new Error(`范围上下限颠倒: ${pathParts.join(".")}`);
+  }
+  Object.entries(value).forEach(([key, item]) =>
+    assertOrderedRanges(item, [...pathParts, key]),
+  );
+}
+
+export function validateSource(file = defaultSourcePath) {
+  const sourceText = fs.readFileSync(file, "utf8");
+  const source = JSON.parse(sourceText);
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(readJson(schemaPath));
+  if (!validate(source)) {
+    throw new Error(
+      `官方目录格式无效: ${ajv.errorsText(validate.errors, { separator: "; " })}`,
+    );
+  }
+  const providerCodes = new Set(source.providers.map((item) => item.code));
+  if (providerCodes.size !== source.providers.length) {
+    throw new Error("官方目录包含重复厂商 code");
+  }
+  const keys = new Set();
+  const identities = new Set();
+  for (const model of source.baseModels) {
+    if (keys.has(model.catalogKey))
+      throw new Error(`重复 catalogKey: ${model.catalogKey}`);
+    keys.add(model.catalogKey);
+    const identity = `${model.alias}\u0000${model.defaultRealModelName}`;
+    if (identities.has(identity))
+      throw new Error(
+        `重复模型身份: ${model.alias} / ${model.defaultRealModelName}`,
+      );
+    identities.add(identity);
+    if (model.providerCode && !providerCodes.has(model.providerCode)) {
+      throw new Error(
+        `模型 ${model.catalogKey} 引用了不存在的厂商 ${model.providerCode}`,
+      );
+    }
+    const capabilityKeys = Object.keys(model.capabilities);
+    const missingCapabilities = model.modelTypes.filter(
+      (type) => !capabilityKeys.includes(type),
+    );
+    const extraCapabilities = capabilityKeys.filter(
+      (type) => !model.modelTypes.includes(type),
+    );
+    if (missingCapabilities.length || extraCapabilities.length) {
+      throw new Error(
+        `模型 ${model.catalogKey} 的 modelTypes 与 capabilities 不一致` +
+          `（缺少: ${missingCapabilities.join(", ") || "无"}；` +
+          `多余: ${extraCapabilities.join(", ") || "无"}）`,
+      );
+    }
+    for (const [type, capability] of Object.entries(model.capabilities)) {
+      if (capability.type !== type) {
+        throw new Error(
+          `模型 ${model.catalogKey} 的能力 ${type}.type 必须等于 ${type}`,
+        );
+      }
+    }
+    const textToSpeech = model.capabilities.text_to_speech;
+    if (textToSpeech) {
+      const voices = Array.isArray(textToSpeech.voices)
+        ? textToSpeech.voices
+        : [];
+      const voiceIds = voices.map((voice) => voice.id);
+      if (new Set(voiceIds).size !== voiceIds.length) {
+        throw new Error(`模型 ${model.catalogKey} 包含重复音色 ID`);
+      }
+      if (
+        textToSpeech.default_voice_id &&
+        !voiceIds.includes(textToSpeech.default_voice_id)
+      ) {
+        throw new Error(
+          `模型 ${model.catalogKey} 的默认音色不在 voices 中: ${textToSpeech.default_voice_id}`,
+        );
+      }
+    }
+    if (model.catalogKey.startsWith("base:seed:")) {
+      if (!model.seedKey)
+        throw new Error(`种子模型 ${model.catalogKey} 必须包含 seedKey`);
+    } else if (model.seedKey !== undefined) {
+      throw new Error(`运行时模型 ${model.catalogKey} 不应包含 seedKey`);
+    }
+    assertOrderedRanges(model.capabilities, [model.catalogKey, "capabilities"]);
+  }
+  const canonicalText = serializeJson(canonicalizeSource(source));
+  if (sourceText !== canonicalText) {
+    throw new Error("官方目录格式不规范，请运行 yarn format 后再提交");
+  }
+  return source;
+}
+
+export function buildRelease(
+  source,
+  { version, note = "", previousVersion = null } = {},
+) {
+  const publishedAt = new Date().toISOString();
+  const selectedVersion = version || publishedAt.replace(/[-:.]/g, "");
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(selectedVersion))
+    throw new Error("目录版本格式无效");
+  const payload = {
+    schemaVersion: source.schemaVersion,
+    providers: source.providers,
+    baseModels: source.baseModels,
+    providerModelPresets: [],
+  };
+  return {
+    version: selectedVersion,
+    publishedAt,
+    generatedAt: publishedAt,
+    ...payload,
+    checksum: checksum(payload),
+    count: source.baseModels.length,
+    ttlSeconds: 21600,
+    previousVersion,
+    sourceRepository: "https://git.51easyai.com/wangbo/easyai",
+    sourcePath: "model-catalog/official/catalog.source.json",
+    ...(note ? { note } : {}),
+  };
+}
+
+export function assertNoMissingModels(
+  previous,
+  source,
+  { allowRemovals = false, removalReason = "" } = {},
+) {
+  const keys = new Set(source.baseModels.map((model) => model.catalogKey));
+  const removed = (previous?.baseModels || [])
+    .filter((model) => !keys.has(model.catalogKey))
+    .map((model) => ({ catalogKey: model.catalogKey, alias: model.alias }));
+  if (removed.length && (!allowRemovals || !removalReason.trim())) {
+    throw new Error(
+      `维护源遗漏已发布模型：${JSON.stringify(removed)}。下线请保留原身份并设置 status=offline；确需移除时须声明 --allow-removals --removal-reason。`,
+    );
+  }
+  return removed;
+}
+
+export function prepareArtifacts(targetDirectory = directory, options = {}) {
+  const source = validateSource(
+    path.join(targetDirectory, "catalog.source.json"),
+  );
+  const removed = assertNoMissingModels(options.previous, source, options);
+  const release = buildRelease(source, {
+    ...options,
+    previousVersion: options.previous?.version || null,
+  });
+  if (removed.length)
+    release.removals = {
+      models: removed,
+      reason: options.removalReason.trim(),
+    };
+  const releasePath = path.join(
+    targetDirectory,
+    "releases",
+    release.version,
+    "catalog.json",
+  );
+  if (fs.existsSync(releasePath))
+    throw new Error("该版本快照已存在，请生成新版本，不能覆盖历史文件");
+  const pointer = {
+    schemaVersion: 1,
+    version: release.version,
+    checksum: release.checksum,
+    publishedAt: release.publishedAt,
+    catalogUrl: `${publicBaseUrl}/releases/${release.version}/catalog.json`,
+  };
+  fs.mkdirSync(path.dirname(releasePath), { recursive: true });
+  fs.writeFileSync(releasePath, serializeJson(release));
+  fs.writeFileSync(
+    path.join(targetDirectory, "latest.json"),
+    serializeJson(pointer),
+  );
+  return { pointer, release };
+}
+
+export function checkArtifacts(targetDirectory = directory) {
+  const source = validateSource(
+    path.join(targetDirectory, "catalog.source.json"),
+  );
+  const pointer = readJson(path.join(targetDirectory, "latest.json"));
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
+    throw new Error("目录版本格式无效");
+  const release = readJson(
+    path.join(targetDirectory, "releases", pointer.version, "catalog.json"),
+  );
+  const payload = {
+    schemaVersion: source.schemaVersion,
+    providers: source.providers,
+    baseModels: source.baseModels,
+    providerModelPresets: [],
+  };
+  if (
+    pointer.version !== release.version ||
+    pointer.checksum !== release.checksum ||
+    release.checksum !== checksum(payload) ||
+    release.checksum !==
+      checksum({
+        schemaVersion: release.schemaVersion,
+        providers: release.providers,
+        baseModels: release.baseModels,
+        providerModelPresets: release.providerModelPresets || [],
+      })
+  ) {
+    throw new Error(
+      "维护源、最新指针和版本快照不一致，请重新准备版本并一并提交",
+    );
+  }
+  if (
+    pointer.catalogUrl !==
+    `${publicBaseUrl}/releases/${release.version}/catalog.json`
+  )
+    throw new Error("目录快照必须指向 easyai 仓库公开地址");
+  return {
+    version: release.version,
+    checksum: release.checksum,
+    baseModelCount: release.baseModels.length,
+    catalogUrl: pointer.catalogUrl,
+  };
+}
+
+function previousCommittedRelease() {
+  const git = (file) =>
+    JSON.parse(
+      execFileSync("git", ["show", `HEAD:model-catalog/official/${file}`], {
+        cwd: repository,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "pipe"],
+        maxBuffer: 4 * 1024 * 1024,
+      }),
+    );
+  let pointer;
+  try {
+    pointer = git("latest.json");
+  } catch (error) {
+    // A missing pointer is only allowed for this repository's first catalog commit.
+    try {
+      execFileSync(
+        "git",
+        ["cat-file", "-e", "HEAD:model-catalog/official/latest.json"],
+        { cwd: repository, stdio: "ignore" },
+      );
+    } catch {
+      return null;
+    }
+    throw error;
+  }
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/.test(pointer.version))
+    throw new Error("已提交目录版本格式无效");
+  return git(`releases/${pointer.version}/catalog.json`);
+}
+
+function main() {
+  const command = args[0] || "validate";
+  if (command === "format") {
+    fs.writeFileSync(
+      defaultSourcePath,
+      serializeJson(canonicalizeSource(readJson(defaultSourcePath))),
+    );
+    validateSource();
+    console.log(JSON.stringify({ action: "formatted" }));
+  } else if (command === "validate") {
+    const source = validateSource();
+    console.log(
+      JSON.stringify({
+        action: "validated",
+        baseModelCount: source.baseModels.length,
+        providerCount: source.providers.length,
+      }),
+    );
+  } else if (command === "prepare") {
+    const result = prepareArtifacts(directory, {
+      previous: previousCommittedRelease(),
+      version: option("version") || undefined,
+      note: option("note"),
+      allowRemovals: args.includes("--allow-removals"),
+      removalReason: option("removal-reason"),
+    });
+    console.log(
+      JSON.stringify(
+        {
+          action: "prepared-for-git-commit",
+          version: result.pointer.version,
+          checksum: result.pointer.checksum,
+          baseModelCount: result.release.count,
+          instruction:
+            "检查 Git 差异后，将维护源、latest.json 和版本快照一并提交并推送；公开 JSON 只来自 Git 已提交内容。",
+        },
+        null,
+        2,
+      ),
+    );
+  } else if (command === "check") {
+    console.log(JSON.stringify(checkArtifacts(), null, 2));
+  } else {
+    throw new Error(
+      "支持 format、validate、prepare、check；目录通过 Git 提交发布，不再使用 OSS/CDN 上传命令。",
+    );
+  }
+}
+
+if (
+  process.argv[1] &&
+  path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
+) {
+  try {
+    main();
+  } catch (error) {
+    console.error(error.message);
+    process.exitCode = 1;
+  }
+}