perf(worker): 按实例内存上限扩展异步容量
将策略推导出的集群目标与单 Worker 内存安全上限分离,避免失活实例的全部容量转移到单个 2 GiB Pod。 生产环境单实例上限和媒体物化并发设为 24;两实例正常时总容量由 16 提升至 48,后续可通过增加 Worker 节点继续扩展。新增 0093 迁移保存实例容量上限,并兼容滚动发布中的旧实例。 风险:更高媒体并发会增加 Worker 和节点内存压力,保留单实例 24 的硬边界并由持续监控观察 RSS、OOM、队列与节点余量。 验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 分配与故障转移测试;迁移安全检查;kubectl kustomize;gofmt -l;git diff --check。
This commit is contained in:
@@ -220,6 +220,10 @@ func (s *Service) startRiverQueue(ctx context.Context, workerEnabled bool) error
|
||||
s.logger.Info("async worker capacity initialized",
|
||||
"capacity", snapshot.Capacity,
|
||||
"desiredCapacity", snapshot.Desired,
|
||||
"activeInstances", snapshot.ActiveInstances,
|
||||
"globalCapacity", snapshot.GlobalCapacity,
|
||||
"instanceID", snapshot.InstanceID,
|
||||
"instanceHardLimit", s.cfg.AsyncWorkerInstanceHardLimit,
|
||||
"hardLimit", snapshot.HardLimit,
|
||||
"enabledModels", snapshot.EnabledModels,
|
||||
"unlimitedModels", snapshot.UnlimitedModels,
|
||||
@@ -289,13 +293,14 @@ func (s *Service) loadAsyncWorkerCapacity(ctx context.Context) (store.AsyncWorke
|
||||
Site: strings.TrimSpace(os.Getenv("EASYAI_SITE")),
|
||||
Revision: strings.TrimSpace(os.Getenv("AI_GATEWAY_REVISION")),
|
||||
DesiredCapacity: snapshot.Capacity,
|
||||
CapacityLimit: s.cfg.AsyncWorkerInstanceHardLimit,
|
||||
HeartbeatStaleAfter: time.Duration(s.cfg.AsyncWorkerRefreshIntervalSeconds) * 6 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return store.AsyncWorkerCapacitySnapshot{}, err
|
||||
}
|
||||
snapshot.Capacity = allocation.Allocated
|
||||
snapshot.GlobalCapacity = allocation.DesiredCapacity
|
||||
snapshot.GlobalCapacity = allocation.GlobalAllocated
|
||||
snapshot.ActiveInstances = allocation.ActiveInstances
|
||||
snapshot.InstanceID = allocation.InstanceID
|
||||
return snapshot, nil
|
||||
@@ -372,7 +377,9 @@ func (s *Service) resizeAsyncWorkerCapacity(ctx context.Context) {
|
||||
"capacity", snapshot.Capacity,
|
||||
"desiredCapacity", snapshot.Desired,
|
||||
"activeInstances", snapshot.ActiveInstances,
|
||||
"globalCapacity", snapshot.GlobalCapacity,
|
||||
"instanceID", snapshot.InstanceID,
|
||||
"instanceHardLimit", s.cfg.AsyncWorkerInstanceHardLimit,
|
||||
"hardLimit", snapshot.HardLimit,
|
||||
"modelDesired", snapshot.ModelDesired,
|
||||
"groupDesired", snapshot.GroupDesired,
|
||||
|
||||
@@ -93,6 +93,9 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...b
|
||||
if cfg.AsyncWorkerHardLimit == 0 {
|
||||
cfg.AsyncWorkerHardLimit = 2048
|
||||
}
|
||||
if cfg.AsyncWorkerInstanceHardLimit == 0 {
|
||||
cfg.AsyncWorkerInstanceHardLimit = 32
|
||||
}
|
||||
if cfg.AsyncWorkerRefreshIntervalSeconds == 0 {
|
||||
cfg.AsyncWorkerRefreshIntervalSeconds = 5
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user