From b51b2d7675a4920076fdcc48471800e6d106b52b Mon Sep 17 00:00:00 2001 From: claude-code-best Date: Wed, 1 Apr 2026 08:36:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=8D=87=E7=BA=A7=20@ant/computer-use-?= =?UTF-8?q?mcp=20=E2=80=94=20=E7=B1=BB=E5=9E=8B=E5=AE=89=E5=85=A8=20stub?= =?UTF-8?q?=20+=20sentinel=20apps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - types.ts: 替换所有 any 为真实类型 (CoordinateMode, CuSubGates, Logger, GrantFlags, CuPermissionRequest/Response, ComputerUseHostAdapter) - index.ts: 所有导出类型化 (DisplayGeometry, FrontmostApp, InstalledApp, RunningApp, ScreenshotResult, CuCallToolResult 等)； targetImageSize() 实现真实缩放逻辑； bindSessionContext() 返回类型正确的空调度器 - sentinelApps.ts: 添加 10 个 macOS 敏感应用 (Terminal, iTerm2, Finder, System Preferences 等) 及其分类 Co-Authored-By: Claude Opus 4.6 --- TODO.md | 4 +- packages/@ant/computer-use-mcp/src/index.ts | 173 ++++++++++++++++-- .../@ant/computer-use-mcp/src/sentinelApps.ts | 33 +++- packages/@ant/computer-use-mcp/src/types.ts | 78 +++++++- 4 files changed, 255 insertions(+), 33 deletions(-) diff --git a/TODO.md b/TODO.md index ce017ac..10e8012 100644 --- a/TODO.md +++ b/TODO.md @@ -10,8 +10,8 @@ - [x] `color-diff-napi` — 颜色差异计算 NAPI 模块 (纯 TS 实现) - [x] `image-processor-napi` — 图像处理 NAPI 模块 (sharp + osascript 剪贴板) - + +- [x] `@ant/computer-use-mcp` — Computer Use MCP 服务 (类型安全 stub + sentinel apps + targetImageSize) - [x] `@ant/computer-use-input` — Computer Use 输入模块 (macOS AppleScript/JXA 实现) diff --git a/packages/@ant/computer-use-mcp/src/index.ts b/packages/@ant/computer-use-mcp/src/index.ts index 25028da..b35f1ef 100644 --- a/packages/@ant/computer-use-mcp/src/index.ts +++ b/packages/@ant/computer-use-mcp/src/index.ts @@ -1,30 +1,163 @@ -export const API_RESIZE_PARAMS: any = {} +/** + * @ant/computer-use-mcp — Stub 实现 + * + * 提供类型安全的 stub，所有函数返回合理的默认值。 + * 在 feature('CHICAGO_MCP') = false 时不会被实际调用， + * 但确保 import 不报错且类型正确。 + */ -export class ComputerExecutor {} +import type { + ComputerUseHostAdapter, + CoordinateMode, + GrantFlags, + Logger, +} from './types' -export type ComputerUseSessionContext = any -export type CuCallToolResult = any -export type CuPermissionRequest = any -export type CuPermissionResponse = any -export const DEFAULT_GRANT_FLAGS: any = {} -export type DisplayGeometry = any -export type FrontmostApp = any -export type InstalledApp = any -export type ResolvePrepareCaptureResult = any -export type RunningApp = any -export type ScreenshotDims = any -export type ScreenshotResult = any +// Re-export types from types.ts +export type { CoordinateMode, Logger } from './types' +export type { + ComputerUseConfig, + ComputerUseHostAdapter, + CuPermissionRequest, + CuPermissionResponse, + CuSubGates, +} from './types' +export { DEFAULT_GRANT_FLAGS } from './types' -export function bindSessionContext(..._args: any[]): any { - return null +// --------------------------------------------------------------------------- +// Types (defined here for callers that import from the main entry) +// --------------------------------------------------------------------------- + +export interface DisplayGeometry { + width: number + height: number + displayId?: number + originX?: number + originY?: number } -export function buildComputerUseTools(..._args: any[]): any[] { +export interface FrontmostApp { + bundleId: string + displayName: string +} + +export interface InstalledApp { + bundleId: string + displayName: string + path: string +} + +export interface RunningApp { + bundleId: string + displayName: string +} + +export interface ScreenshotResult { + base64: string + width: number + height: number +} + +export type ResolvePrepareCaptureResult = ScreenshotResult + +export interface ScreenshotDims { + width: number + height: number + displayWidth: number + displayHeight: number + displayId: number + originX: number + originY: number +} + +export interface CuCallToolResultContent { + type: 'image' | 'text' + data?: string + mimeType?: string + text?: string +} + +export interface CuCallToolResult { + content: CuCallToolResultContent[] + telemetry: { + error_kind?: string + [key: string]: unknown + } +} + +export type ComputerUseSessionContext = Record + +// --------------------------------------------------------------------------- +// API_RESIZE_PARAMS — 默认的截图缩放参数 +// --------------------------------------------------------------------------- + +export const API_RESIZE_PARAMS = { + maxWidth: 1280, + maxHeight: 800, + maxPixels: 1280 * 800, +} + +// --------------------------------------------------------------------------- +// ComputerExecutor — stub class +// --------------------------------------------------------------------------- + +export class ComputerExecutor { + capabilities: Record = {} +} + +// --------------------------------------------------------------------------- +// Functions — 返回合理默认值的 stub +// --------------------------------------------------------------------------- + +/** + * 计算目标截图尺寸。 + * 在物理宽高和 API 限制之间取最优尺寸。 + */ +export function targetImageSize( + physW: number, + physH: number, + _params?: typeof API_RESIZE_PARAMS, +): [number, number] { + const maxW = _params?.maxWidth ?? 1280 + const maxH = _params?.maxHeight ?? 800 + const scale = Math.min(1, maxW / physW, maxH / physH) + return [Math.round(physW * scale), Math.round(physH * scale)] +} + +/** + * 绑定会话上下文，返回工具调度函数。 + * Stub 返回一个始终返回空结果的调度器。 + */ +export function bindSessionContext( + _adapter: ComputerUseHostAdapter, + _coordinateMode: CoordinateMode, + _ctx: ComputerUseSessionContext, +): (name: string, args: unknown) => Promise { + return async (_name: string, _args: unknown) => ({ + content: [], + telemetry: {}, + }) +} + +/** + * 构建 Computer Use 工具定义列表。 + * Stub 返回空数组（无工具）。 + */ +export function buildComputerUseTools( + _capabilities?: Record, + _coordinateMode?: CoordinateMode, + _installedAppNames?: string[], +): Array<{ name: string; description: string; inputSchema: Record }> { return [] } -export function createComputerUseMcpServer(..._args: any[]): any { +/** + * 创建 Computer Use MCP server。 + * Stub 返回 null（服务未启用）。 + */ +export function createComputerUseMcpServer( + _adapter?: ComputerUseHostAdapter, + _coordinateMode?: CoordinateMode, +): null { return null } - -export const targetImageSize: any = null diff --git a/packages/@ant/computer-use-mcp/src/sentinelApps.ts b/packages/@ant/computer-use-mcp/src/sentinelApps.ts index de150b3..27a67a1 100644 --- a/packages/@ant/computer-use-mcp/src/sentinelApps.ts +++ b/packages/@ant/computer-use-mcp/src/sentinelApps.ts @@ -1,5 +1,32 @@ -export const sentinelApps: string[] = [] +/** + * Sentinel apps — 需要特殊权限警告的应用列表 + * + * 包含终端、文件管理器、系统设置等敏感应用。 + * Computer Use 操作这些应用时会显示额外警告。 + */ -export function getSentinelCategory(_appName: string): string | null { - return null +type SentinelCategory = 'shell' | 'filesystem' | 'system_settings' + +const SENTINEL_MAP: Record = { + // Shell / Terminal + 'com.apple.Terminal': 'shell', + 'com.googlecode.iterm2': 'shell', + 'dev.warp.Warp-Stable': 'shell', + 'io.alacritty': 'shell', + 'com.github.wez.wezterm': 'shell', + 'net.kovidgoyal.kitty': 'shell', + 'co.zeit.hyper': 'shell', + + // Filesystem + 'com.apple.finder': 'filesystem', + + // System Settings + 'com.apple.systempreferences': 'system_settings', + 'com.apple.SystemPreferences': 'system_settings', +} + +export const sentinelApps: string[] = Object.keys(SENTINEL_MAP) + +export function getSentinelCategory(bundleId: string): SentinelCategory | null { + return SENTINEL_MAP[bundleId] ?? null } diff --git a/packages/@ant/computer-use-mcp/src/types.ts b/packages/@ant/computer-use-mcp/src/types.ts index 4fc112c..2247360 100644 --- a/packages/@ant/computer-use-mcp/src/types.ts +++ b/packages/@ant/computer-use-mcp/src/types.ts @@ -1,8 +1,70 @@ -export type ComputerUseConfig = any -export type ComputerUseHostAdapter = any -export type CoordinateMode = any -export type CuPermissionRequest = any -export type CuPermissionResponse = any -export type CuSubGates = any -export const DEFAULT_GRANT_FLAGS: any = {} -export type Logger = any +/** + * @ant/computer-use-mcp — Types + * + * 从调用侧反推的真实类型定义，替代 any stub。 + */ + +export type CoordinateMode = 'pixels' | 'normalized' + +export interface CuSubGates { + pixelValidation: boolean + clipboardPasteMultiline: boolean + mouseAnimation: boolean + hideBeforeAction: boolean + autoTargetDisplay: boolean + clipboardGuard: boolean +} + +export interface Logger { + silly(message: string, ...args: unknown[]): void + debug(message: string, ...args: unknown[]): void + info(message: string, ...args: unknown[]): void + warn(message: string, ...args: unknown[]): void + error(message: string, ...args: unknown[]): void +} + +export interface CuPermissionRequest { + apps: Array<{ bundleId: string; displayName: string }> + requestedFlags: GrantFlags + reason: string + tccState: { accessibility: boolean; screenRecording: boolean } + willHide: string[] +} + +export interface GrantFlags { + clipboardRead: boolean + clipboardWrite: boolean + systemKeyCombos: boolean +} + +export interface CuPermissionResponse { + granted: string[] + denied: string[] + flags: GrantFlags +} + +export const DEFAULT_GRANT_FLAGS: GrantFlags = { + clipboardRead: false, + clipboardWrite: false, + systemKeyCombos: false, +} + +export interface ComputerUseConfig { + coordinateMode: CoordinateMode + enabledTools: string[] +} + +export interface ComputerUseHostAdapter { + serverName: string + logger: Logger + executor: ComputerExecutor + ensureOsPermissions(): Promise<{ granted: true } | { granted: false; accessibility: boolean; screenRecording: boolean }> + isDisabled(): boolean + getSubGates(): CuSubGates + getAutoUnhideEnabled(): boolean + cropRawPatch?(base64: string, x: number, y: number, w: number, h: number): Promise +} + +export interface ComputerExecutor { + capabilities: Record +}