feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
+16
-4
@@ -37,6 +37,7 @@ import type {
|
||||
UserGroupUpsertRequest,
|
||||
UserGroup,
|
||||
WalletRechargeRequest,
|
||||
WorkerClusterRuntime,
|
||||
} from '@easyai-ai-gateway/contracts';
|
||||
import {
|
||||
batchAccessRules,
|
||||
@@ -67,6 +68,7 @@ import {
|
||||
getRunnerPolicy,
|
||||
getSecurityEventConnection,
|
||||
getWalletSummary,
|
||||
getWorkerClusterRuntime,
|
||||
listAccessRules,
|
||||
listAdminTasks,
|
||||
listAuditLogs,
|
||||
@@ -201,6 +203,7 @@ type DataKey =
|
||||
| 'runtimePolicySets'
|
||||
| 'rateLimitWindows'
|
||||
| 'modelRateLimits'
|
||||
| 'workerClusterRuntime'
|
||||
| 'tenants'
|
||||
| 'users'
|
||||
| 'userGroups'
|
||||
@@ -255,6 +258,7 @@ export function App() {
|
||||
const [rateLimitWindows, setRateLimitWindows] = useState<RateLimitWindow[]>([]);
|
||||
const [modelRateLimits, setModelRateLimits] = useState<ModelRateLimitStatus[]>([]);
|
||||
const [modelRateLimitsUpdatedAt, setModelRateLimitsUpdatedAt] = useState<number | null>(null);
|
||||
const [workerClusterRuntime, setWorkerClusterRuntime] = useState<WorkerClusterRuntime | null>(null);
|
||||
const [tenants, setTenants] = useState<GatewayTenant[]>([]);
|
||||
const [users, setUsers] = useState<GatewayUser[]>([]);
|
||||
const [userGroups, setUserGroups] = useState<UserGroup[]>([]);
|
||||
@@ -381,18 +385,21 @@ export function App() {
|
||||
useEffect(() => {
|
||||
if (!token || activePage !== 'admin' || adminSection !== 'realtimeLoad') return undefined;
|
||||
const timer = window.setInterval(() => {
|
||||
void Promise.all([listModelRateLimitStatuses(token), listPlatforms(token)])
|
||||
.then(([rateLimitResponse, platformResponse]) => {
|
||||
void Promise.all([listModelRateLimitStatuses(token), listPlatforms(token), getWorkerClusterRuntime(token)])
|
||||
.then(([rateLimitResponse, platformResponse, workerRuntime]) => {
|
||||
setModelRateLimits(rateLimitResponse.items);
|
||||
setModelRateLimitsUpdatedAt(Date.now());
|
||||
setPlatforms(platformResponse.items);
|
||||
setWorkerClusterRuntime(workerRuntime);
|
||||
loadedDataKeysRef.current.add('modelRateLimits');
|
||||
loadedDataKeysRef.current.add('platforms');
|
||||
loadedDataKeysRef.current.add('workerClusterRuntime');
|
||||
})
|
||||
.catch((err) => {
|
||||
if (handleAuthExpired(err, token)) return;
|
||||
loadedDataKeysRef.current.delete('modelRateLimits');
|
||||
loadedDataKeysRef.current.delete('platforms');
|
||||
loadedDataKeysRef.current.delete('workerClusterRuntime');
|
||||
});
|
||||
}, 3000);
|
||||
return () => window.clearInterval(timer);
|
||||
@@ -446,6 +453,7 @@ export function App() {
|
||||
rateLimitWindows,
|
||||
modelRateLimits,
|
||||
modelRateLimitsUpdatedAt,
|
||||
workerClusterRuntime,
|
||||
runtimePolicySets,
|
||||
securityEventConnection,
|
||||
taskResult,
|
||||
@@ -455,7 +463,7 @@ export function App() {
|
||||
users,
|
||||
walletAccounts,
|
||||
walletTransactions,
|
||||
}), [accessRules, adminTasks, apiKeys, auditLogs, baseModels, clientCustomizationSettings, currentUser, currentUserGroups, fileStorageChannels, fileStorageSettings, modelCatalog, modelRateLimits, modelRateLimitsUpdatedAt, models, networkProxyConfig, platforms, pricingRuleSets, pricingRules, providers, rateLimitWindows, runnerPolicy, runtimePolicySets, securityEventConnection, taskResult, tasks, tenants, userGroups, users, walletAccounts, walletTransactions]);
|
||||
}), [accessRules, adminTasks, apiKeys, auditLogs, baseModels, clientCustomizationSettings, currentUser, currentUserGroups, fileStorageChannels, fileStorageSettings, modelCatalog, modelRateLimits, modelRateLimitsUpdatedAt, models, networkProxyConfig, platforms, pricingRuleSets, pricingRules, providers, rateLimitWindows, runnerPolicy, runtimePolicySets, securityEventConnection, taskResult, tasks, tenants, userGroups, users, walletAccounts, walletTransactions, workerClusterRuntime]);
|
||||
|
||||
async function refresh(nextToken = token) {
|
||||
await ensureRouteData(nextToken, true);
|
||||
@@ -593,6 +601,9 @@ export function App() {
|
||||
setModelRateLimitsUpdatedAt(Date.now());
|
||||
}
|
||||
return;
|
||||
case 'workerClusterRuntime':
|
||||
setWorkerClusterRuntime(await getWorkerClusterRuntime(nextToken));
|
||||
return;
|
||||
case 'tenants':
|
||||
setTenants((await listTenants(nextToken)).items);
|
||||
return;
|
||||
@@ -1255,6 +1266,7 @@ export function App() {
|
||||
setAuditLogs([]);
|
||||
setRateLimitWindows([]);
|
||||
setModelRateLimits([]);
|
||||
setWorkerClusterRuntime(null);
|
||||
setTenants([]);
|
||||
setUsers([]);
|
||||
setUserGroups([]);
|
||||
@@ -1744,7 +1756,7 @@ function dataKeysForRoute(
|
||||
case 'platforms':
|
||||
return ['platforms', 'models', 'providers', 'baseModels', 'pricingRuleSets', 'networkProxyConfig'];
|
||||
case 'realtimeLoad':
|
||||
return ['platforms', 'modelRateLimits'];
|
||||
return ['platforms', 'modelRateLimits', 'workerClusterRuntime'];
|
||||
case 'tasks':
|
||||
return ['adminTasks', 'tenants', 'users', 'userGroups', 'platforms', 'models'];
|
||||
case 'tenants':
|
||||
|
||||
@@ -59,6 +59,7 @@ import type {
|
||||
WalletBalanceAdjustmentRequest,
|
||||
WalletRechargeRequest,
|
||||
WalletSummaryResponse,
|
||||
WorkerClusterRuntime,
|
||||
} from '@easyai-ai-gateway/contracts';
|
||||
import type { AdminTaskQuery, PlatformCreateInput, PlatformModelBindingInput, WorkspaceTaskQuery } from './types';
|
||||
|
||||
@@ -1037,6 +1038,10 @@ export async function listModelRateLimitStatuses(token: string): Promise<ListRes
|
||||
return request<ListResponse<ModelRateLimitStatus>>('/api/admin/runtime/model-rate-limits', { token });
|
||||
}
|
||||
|
||||
export async function getWorkerClusterRuntime(token: string): Promise<WorkerClusterRuntime> {
|
||||
return request<WorkerClusterRuntime>('/api/admin/runtime/workers', { token });
|
||||
}
|
||||
|
||||
export async function restoreModelRuntimeStatus(token: string, platformModelId: string): Promise<ModelRateLimitStatus> {
|
||||
return request<ModelRateLimitStatus>(`/api/admin/runtime/model-rate-limits/${platformModelId}/restore`, {
|
||||
method: 'POST',
|
||||
|
||||
@@ -26,6 +26,7 @@ import type {
|
||||
RuntimePolicySet,
|
||||
SecurityEventConnectionResponse,
|
||||
UserGroup,
|
||||
WorkerClusterRuntime,
|
||||
} from '@easyai-ai-gateway/contracts';
|
||||
|
||||
export interface ConsoleData {
|
||||
@@ -50,6 +51,7 @@ export interface ConsoleData {
|
||||
rateLimitWindows: RateLimitWindow[];
|
||||
modelRateLimits: ModelRateLimitStatus[];
|
||||
modelRateLimitsUpdatedAt: number | null;
|
||||
workerClusterRuntime: WorkerClusterRuntime | null;
|
||||
runtimePolicySets: RuntimePolicySet[];
|
||||
securityEventConnection: SecurityEventConnectionResponse | null;
|
||||
taskResult: GatewayTask | null;
|
||||
|
||||
@@ -50,7 +50,7 @@ export const adminPages = [
|
||||
{ title: '用户组策略', path: '/admin/user-groups', description: '用户组成员、充值折扣、调用折扣、TPM/RPM/并发和队列优先级。' },
|
||||
{ title: '全局模型配置', path: '/admin/models/global', description: '基准模型库、能力 schema、基准定价和默认限流模板。' },
|
||||
{ title: '平台管理', path: '/admin/platforms', description: '平台 CRUD、凭证、默认折扣、平台模型、限流和重试策略。' },
|
||||
{ title: '实时负载', path: '/admin/realtime-load', description: '按平台模型查看实时 RPM、TPM、并发、排队和冷却状态。' },
|
||||
{ title: '实时负载', path: '/admin/realtime-load', description: '查看平台模型 RPM、TPM、并发，以及 Worker 自适应容量和压力状态。' },
|
||||
{ title: '任务记录', path: '/admin/tasks', description: '跨租户查询任务、执行链路、参数转换、计费和原始详情。' },
|
||||
{ title: '计费结算', path: '/admin/billing-settlements', description: '查询计费结算队列，批量处理等待重试和人工复核记录。' },
|
||||
{ title: '运行与队列', path: '/admin/runtime/queues', description: 'TPM/RPM 窗口、并发 lease、cooldown、任务恢复和队列积压。' },
|
||||
|
||||
@@ -179,6 +179,7 @@ export function AdminPage(props: {
|
||||
modelRateLimits={props.data.modelRateLimits}
|
||||
modelRateLimitsUpdatedAt={props.data.modelRateLimitsUpdatedAt}
|
||||
platforms={props.data.platforms}
|
||||
workerClusterRuntime={props.data.workerClusterRuntime}
|
||||
onSavePlatformDynamicPriority={props.onSavePlatformDynamicPriority}
|
||||
onRestoreRuntimeModel={props.onRestoreRuntimeModel}
|
||||
/>
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { useEffect, useMemo, useState, type FormEvent } from 'react';
|
||||
import { Popover as AntPopover } from 'antd';
|
||||
import { CheckCircle2, History, RotateCcw, Search, SlidersHorizontal } from 'lucide-react';
|
||||
import type { IntegrationPlatform, ModelRateLimitStatus, PlatformDynamicPriorityUpdateRequest, PlatformPolicyEvent, PriorityDemotionRecord } from '@easyai-ai-gateway/contracts';
|
||||
import type { IntegrationPlatform, ModelRateLimitStatus, PlatformDynamicPriorityUpdateRequest, PlatformPolicyEvent, PriorityDemotionRecord, WorkerClusterRuntime, WorkerInstanceRuntime } from '@easyai-ai-gateway/contracts';
|
||||
import { Badge, Button, Card, CardContent, EmptyState, FormDialog, Input, Label, Select, Table, TableCell, TableHead, TableRow } from '../../components/ui';
|
||||
|
||||
export function RealtimeLoadPanel(props: {
|
||||
modelRateLimits: ModelRateLimitStatus[];
|
||||
modelRateLimitsUpdatedAt: number | null;
|
||||
platforms: IntegrationPlatform[];
|
||||
workerClusterRuntime: WorkerClusterRuntime | null;
|
||||
onSavePlatformDynamicPriority: (platformId: string, input: PlatformDynamicPriorityUpdateRequest) => Promise<void>;
|
||||
onRestoreRuntimeModel: (platformModelId: string) => Promise<void>;
|
||||
}) {
|
||||
@@ -119,6 +120,7 @@ export function RealtimeLoadPanel(props: {
|
||||
|
||||
return (
|
||||
<section className="pageStack">
|
||||
<WorkerRuntimeTable runtime={props.workerClusterRuntime} />
|
||||
<Card className="compactAdminTableCard">
|
||||
<CardContent className="compactAdminTableContent">
|
||||
<div className="compactAdminToolbar realtimeCompactToolbar">
|
||||
@@ -241,6 +243,95 @@ export function RealtimeLoadPanel(props: {
|
||||
);
|
||||
}
|
||||
|
||||
function WorkerRuntimeTable(props: { runtime: WorkerClusterRuntime | null }) {
|
||||
const workers = props.runtime?.workers ?? [];
|
||||
const queue = props.runtime?.queue;
|
||||
return (
|
||||
<Card className="compactAdminTableCard">
|
||||
<CardContent className="compactAdminTableContent">
|
||||
<div className="compactAdminToolbar">
|
||||
<span className="platformTableName">
|
||||
<strong>Worker 自适应负载</strong>
|
||||
<small>
|
||||
{queue
|
||||
? `共享队列 ${queue.queued}，运行 ${queue.running}，最老等待 ${Math.round(queue.oldestWaitSeconds)} 秒`
|
||||
: '等待集群负载快照'}
|
||||
</small>
|
||||
</span>
|
||||
</div>
|
||||
{!workers.length ? (
|
||||
<EmptyState title="暂无活跃 Worker" description="Worker 心跳后会显示安全容量、阶段分布和压力状态。" />
|
||||
) : (
|
||||
<div className="platformLimitTableViewport">
|
||||
<Table className="platformDataTable platformLimitTable" density="compact">
|
||||
<TableRow className="shTableHeader">
|
||||
<TableHead>Worker</TableHead>
|
||||
<TableHead>压力</TableHead>
|
||||
<TableHead className="platformLimitNumberHead">领取 / 安全 / Hard</TableHead>
|
||||
<TableHead className="platformLimitNumberHead">重负载许可</TableHead>
|
||||
<TableHead className="platformLimitNumberHead">准备 / 等待 / 收尾</TableHead>
|
||||
<TableHead className="platformLimitNumberHead">任务 / 厂商租约</TableHead>
|
||||
<TableHead>最近采样</TableHead>
|
||||
</TableRow>
|
||||
{workers.map((worker) => (
|
||||
<TableRow key={worker.instanceId}>
|
||||
<TableCell>
|
||||
<span className="platformTableName">
|
||||
<strong>{worker.podName || worker.instanceId}</strong>
|
||||
<small>{[worker.site, shortId(worker.revision)].filter(Boolean).join(' · ') || shortId(worker.instanceId)}</small>
|
||||
</span>
|
||||
</TableCell>
|
||||
<TableCell>{workerPressureCell(worker)}</TableCell>
|
||||
<TableCell className="platformLimitNumberCell">
|
||||
<strong>{worker.allocatedCapacity} / {worker.safeCapacity} / {worker.hardCapacityLimit}</strong>
|
||||
</TableCell>
|
||||
<TableCell className="platformLimitNumberCell">{worker.heavyCapacity}</TableCell>
|
||||
<TableCell className="platformLimitNumberCell">
|
||||
<span className="rateMetricCell">
|
||||
<strong>{worker.preparingTasks} / {worker.waitingUpstreamTasks} / {worker.finalizingTasks}</strong>
|
||||
<small>上报活跃 {worker.reportedActiveTasks}</small>
|
||||
</span>
|
||||
</TableCell>
|
||||
<TableCell className="platformLimitNumberCell">{worker.runningTasks} / {worker.activeLeases}</TableCell>
|
||||
<TableCell>
|
||||
<span className="platformTableName">
|
||||
<strong>{formatDateTime(worker.loadSampledAt) || '-'}</strong>
|
||||
<small>心跳 {formatDateTime(worker.heartbeatAt)}</small>
|
||||
</span>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</Table>
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
function workerPressureCell(worker: WorkerInstanceRuntime) {
|
||||
const variant = worker.pressureState === 'critical'
|
||||
? 'destructive'
|
||||
: worker.pressureState === 'busy'
|
||||
? 'warning'
|
||||
: worker.pressureState === 'normal'
|
||||
? 'success'
|
||||
: 'secondary';
|
||||
const label = worker.pressureState === 'critical'
|
||||
? '临界'
|
||||
: worker.pressureState === 'busy'
|
||||
? '繁忙'
|
||||
: worker.pressureState === 'normal'
|
||||
? '正常'
|
||||
: '未知';
|
||||
return (
|
||||
<span className="platformTableName">
|
||||
<strong><Badge variant={variant}>{label}</Badge></strong>
|
||||
<small>{worker.pressureReason || '无压力原因'}</small>
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
type PriorityDialogState = {
|
||||
platform: IntegrationPlatform | undefined;
|
||||
status: ModelRateLimitStatus;
|
||||
|
||||
Reference in New Issue
Block a user