perf(worker): 按实例内存上限扩展异步容量
将策略推导出的集群目标与单 Worker 内存安全上限分离,避免失活实例的全部容量转移到单个 2 GiB Pod。 生产环境单实例上限和媒体物化并发设为 24;两实例正常时总容量由 16 提升至 48,后续可通过增加 Worker 节点继续扩展。新增 0093 迁移保存实例容量上限,并兼容滚动发布中的旧实例。 风险:更高媒体并发会增加 Worker 和节点内存压力,保留单实例 24 的硬边界并由持续监控观察 RSS、OOM、队列与节点余量。 验证:go test ./... -count=1;go vet ./...;真实 PostgreSQL 分配与故障转移测试;迁移安全检查;kubectl kustomize;gofmt -l;git diff --check。
This commit is contained in:
@@ -63,6 +63,7 @@ type Config struct {
|
||||
MediaMaterializationConcurrency int
|
||||
AsyncQueueWorkerEnabled bool
|
||||
AsyncWorkerHardLimit int
|
||||
AsyncWorkerInstanceHardLimit int
|
||||
AsyncWorkerRefreshIntervalSeconds int
|
||||
}
|
||||
|
||||
@@ -123,6 +124,7 @@ func Load() Config {
|
||||
MediaMaterializationConcurrency: envIntValidated("AI_GATEWAY_MEDIA_MATERIALIZATION_CONCURRENCY", 8),
|
||||
AsyncQueueWorkerEnabled: env("AI_GATEWAY_ASYNC_QUEUE_WORKER_ENABLED", "true") == "true",
|
||||
AsyncWorkerHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", 2048),
|
||||
AsyncWorkerInstanceHardLimit: envIntValidated("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", 32),
|
||||
AsyncWorkerRefreshIntervalSeconds: envIntValidated("AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS", 5),
|
||||
}
|
||||
}
|
||||
@@ -150,6 +152,9 @@ func (c Config) Validate() error {
|
||||
if c.AsyncWorkerHardLimit < 1 || c.AsyncWorkerHardLimit > 10000 {
|
||||
return errors.New("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT must be between 1 and 10000")
|
||||
}
|
||||
if c.AsyncWorkerInstanceHardLimit < 1 || c.AsyncWorkerInstanceHardLimit > 10000 {
|
||||
return errors.New("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT must be between 1 and 10000")
|
||||
}
|
||||
if c.AsyncWorkerRefreshIntervalSeconds < 1 {
|
||||
return errors.New("AI_GATEWAY_ASYNC_WORKER_REFRESH_INTERVAL_SECONDS must be positive")
|
||||
}
|
||||
|
||||
@@ -26,7 +26,12 @@ func TestLoadIdentitySecretStoreUsesNewEnvironmentNamesAndIgnoresLegacyBusinessV
|
||||
}
|
||||
|
||||
func TestValidateIdentityFileSecretStoreRequiresDirectory(t *testing.T) {
|
||||
cfg := Config{IdentitySecretStore: "file", AsyncWorkerHardLimit: 2048, AsyncWorkerRefreshIntervalSeconds: 5}
|
||||
cfg := Config{
|
||||
IdentitySecretStore: "file",
|
||||
AsyncWorkerHardLimit: 2048,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 5,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "IDENTITY_SECRET_DIR") {
|
||||
t.Fatalf("Validate() error = %v, want missing identity secret directory", err)
|
||||
}
|
||||
@@ -41,6 +46,7 @@ func TestValidateIdentityKubernetesSecretStore(t *testing.T) {
|
||||
IdentitySecretStore: "kubernetes",
|
||||
IdentityKubernetesSecretName: "easyai-gateway-identity",
|
||||
AsyncWorkerHardLimit: 2048,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 5,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "namespace") {
|
||||
@@ -58,6 +64,7 @@ func TestValidateIdentitySecurityEventTiming(t *testing.T) {
|
||||
IdentitySecurityEventStaleAfterSeconds: 60,
|
||||
IdentitySecurityEventClockSkewSeconds: 60,
|
||||
AsyncWorkerHardLimit: 2048,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 5,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "heartbeat") {
|
||||
@@ -66,11 +73,20 @@ func TestValidateIdentitySecurityEventTiming(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestValidateAsyncWorkerSettings(t *testing.T) {
|
||||
cfg := Config{AsyncWorkerHardLimit: 10001, AsyncWorkerRefreshIntervalSeconds: 5}
|
||||
cfg := Config{
|
||||
AsyncWorkerHardLimit: 10001,
|
||||
AsyncWorkerInstanceHardLimit: 32,
|
||||
AsyncWorkerRefreshIntervalSeconds: 5,
|
||||
}
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "HARD_LIMIT") {
|
||||
t.Fatalf("Validate() error = %v, want invalid hard limit", err)
|
||||
}
|
||||
cfg.AsyncWorkerHardLimit = 2048
|
||||
cfg.AsyncWorkerInstanceHardLimit = 10001
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "INSTANCE_HARD_LIMIT") {
|
||||
t.Fatalf("Validate() error = %v, want invalid instance hard limit", err)
|
||||
}
|
||||
cfg.AsyncWorkerInstanceHardLimit = 32
|
||||
cfg.AsyncWorkerRefreshIntervalSeconds = 0
|
||||
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "REFRESH_INTERVAL") {
|
||||
t.Fatalf("Validate() error = %v, want invalid refresh interval", err)
|
||||
@@ -80,6 +96,12 @@ func TestValidateAsyncWorkerSettings(t *testing.T) {
|
||||
if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "HARD_LIMIT") {
|
||||
t.Fatalf("Validate() error = %v, want invalid non-integer hard limit", err)
|
||||
}
|
||||
t.Setenv("AI_GATEWAY_ASYNC_WORKER_HARD_LIMIT", "2048")
|
||||
t.Setenv("AI_GATEWAY_ASYNC_WORKER_INSTANCE_HARD_LIMIT", "not-an-integer")
|
||||
loaded = Load()
|
||||
if err := loaded.Validate(); err == nil || !strings.Contains(err.Error(), "INSTANCE_HARD_LIMIT") {
|
||||
t.Fatalf("Validate() error = %v, want invalid non-integer instance hard limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAsyncQueueWorkerEnabled(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user