feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
+16
-4
@@ -37,6 +37,7 @@ import type {
|
||||
UserGroupUpsertRequest,
|
||||
UserGroup,
|
||||
WalletRechargeRequest,
|
||||
WorkerClusterRuntime,
|
||||
} from '@easyai-ai-gateway/contracts';
|
||||
import {
|
||||
batchAccessRules,
|
||||
@@ -67,6 +68,7 @@ import {
|
||||
getRunnerPolicy,
|
||||
getSecurityEventConnection,
|
||||
getWalletSummary,
|
||||
getWorkerClusterRuntime,
|
||||
listAccessRules,
|
||||
listAdminTasks,
|
||||
listAuditLogs,
|
||||
@@ -201,6 +203,7 @@ type DataKey =
|
||||
| 'runtimePolicySets'
|
||||
| 'rateLimitWindows'
|
||||
| 'modelRateLimits'
|
||||
| 'workerClusterRuntime'
|
||||
| 'tenants'
|
||||
| 'users'
|
||||
| 'userGroups'
|
||||
@@ -255,6 +258,7 @@ export function App() {
|
||||
const [rateLimitWindows, setRateLimitWindows] = useState<RateLimitWindow[]>([]);
|
||||
const [modelRateLimits, setModelRateLimits] = useState<ModelRateLimitStatus[]>([]);
|
||||
const [modelRateLimitsUpdatedAt, setModelRateLimitsUpdatedAt] = useState<number | null>(null);
|
||||
const [workerClusterRuntime, setWorkerClusterRuntime] = useState<WorkerClusterRuntime | null>(null);
|
||||
const [tenants, setTenants] = useState<GatewayTenant[]>([]);
|
||||
const [users, setUsers] = useState<GatewayUser[]>([]);
|
||||
const [userGroups, setUserGroups] = useState<UserGroup[]>([]);
|
||||
@@ -381,18 +385,21 @@ export function App() {
|
||||
useEffect(() => {
|
||||
if (!token || activePage !== 'admin' || adminSection !== 'realtimeLoad') return undefined;
|
||||
const timer = window.setInterval(() => {
|
||||
void Promise.all([listModelRateLimitStatuses(token), listPlatforms(token)])
|
||||
.then(([rateLimitResponse, platformResponse]) => {
|
||||
void Promise.all([listModelRateLimitStatuses(token), listPlatforms(token), getWorkerClusterRuntime(token)])
|
||||
.then(([rateLimitResponse, platformResponse, workerRuntime]) => {
|
||||
setModelRateLimits(rateLimitResponse.items);
|
||||
setModelRateLimitsUpdatedAt(Date.now());
|
||||
setPlatforms(platformResponse.items);
|
||||
setWorkerClusterRuntime(workerRuntime);
|
||||
loadedDataKeysRef.current.add('modelRateLimits');
|
||||
loadedDataKeysRef.current.add('platforms');
|
||||
loadedDataKeysRef.current.add('workerClusterRuntime');
|
||||
})
|
||||
.catch((err) => {
|
||||
if (handleAuthExpired(err, token)) return;
|
||||
loadedDataKeysRef.current.delete('modelRateLimits');
|
||||
loadedDataKeysRef.current.delete('platforms');
|
||||
loadedDataKeysRef.current.delete('workerClusterRuntime');
|
||||
});
|
||||
}, 3000);
|
||||
return () => window.clearInterval(timer);
|
||||
@@ -446,6 +453,7 @@ export function App() {
|
||||
rateLimitWindows,
|
||||
modelRateLimits,
|
||||
modelRateLimitsUpdatedAt,
|
||||
workerClusterRuntime,
|
||||
runtimePolicySets,
|
||||
securityEventConnection,
|
||||
taskResult,
|
||||
@@ -455,7 +463,7 @@ export function App() {
|
||||
users,
|
||||
walletAccounts,
|
||||
walletTransactions,
|
||||
}), [accessRules, adminTasks, apiKeys, auditLogs, baseModels, clientCustomizationSettings, currentUser, currentUserGroups, fileStorageChannels, fileStorageSettings, modelCatalog, modelRateLimits, modelRateLimitsUpdatedAt, models, networkProxyConfig, platforms, pricingRuleSets, pricingRules, providers, rateLimitWindows, runnerPolicy, runtimePolicySets, securityEventConnection, taskResult, tasks, tenants, userGroups, users, walletAccounts, walletTransactions]);
|
||||
}), [accessRules, adminTasks, apiKeys, auditLogs, baseModels, clientCustomizationSettings, currentUser, currentUserGroups, fileStorageChannels, fileStorageSettings, modelCatalog, modelRateLimits, modelRateLimitsUpdatedAt, models, networkProxyConfig, platforms, pricingRuleSets, pricingRules, providers, rateLimitWindows, runnerPolicy, runtimePolicySets, securityEventConnection, taskResult, tasks, tenants, userGroups, users, walletAccounts, walletTransactions, workerClusterRuntime]);
|
||||
|
||||
async function refresh(nextToken = token) {
|
||||
await ensureRouteData(nextToken, true);
|
||||
@@ -593,6 +601,9 @@ export function App() {
|
||||
setModelRateLimitsUpdatedAt(Date.now());
|
||||
}
|
||||
return;
|
||||
case 'workerClusterRuntime':
|
||||
setWorkerClusterRuntime(await getWorkerClusterRuntime(nextToken));
|
||||
return;
|
||||
case 'tenants':
|
||||
setTenants((await listTenants(nextToken)).items);
|
||||
return;
|
||||
@@ -1255,6 +1266,7 @@ export function App() {
|
||||
setAuditLogs([]);
|
||||
setRateLimitWindows([]);
|
||||
setModelRateLimits([]);
|
||||
setWorkerClusterRuntime(null);
|
||||
setTenants([]);
|
||||
setUsers([]);
|
||||
setUserGroups([]);
|
||||
@@ -1744,7 +1756,7 @@ function dataKeysForRoute(
|
||||
case 'platforms':
|
||||
return ['platforms', 'models', 'providers', 'baseModels', 'pricingRuleSets', 'networkProxyConfig'];
|
||||
case 'realtimeLoad':
|
||||
return ['platforms', 'modelRateLimits'];
|
||||
return ['platforms', 'modelRateLimits', 'workerClusterRuntime'];
|
||||
case 'tasks':
|
||||
return ['adminTasks', 'tenants', 'users', 'userGroups', 'platforms', 'models'];
|
||||
case 'tenants':
|
||||
|
||||
Reference in New Issue
Block a user