BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(routing): 引入多执行池智能调度

Browse Source 
将 Worker 发现、路由画像、容量与执行传输抽象为平台无关接口，新增 Kubernetes 和静态容量适配器，并以 shadow 模式接入生产配置。

实现网络与容量评分、路由防抖、池队列、同步 Worker 租约、一次性执行令牌，以及提交状态不明时禁止重复分配的安全语义。

新增 0105 兼容迁移、管理接口、指标、OpenAPI 和回归测试。已执行全量 Go 测试、go vet、OpenAPI、迁移安全、Compose 与 Kustomize 验证。
This commit is contained in:
wangbo
2026-08-05 22:25:37 +08:00
parent 03c0873649
commit 7786692d32
58 changed files with 4510 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/internal/config/config.go
+142 -31
View File
@@ -1,6 +1,7 @@
package config
import (
"encoding/json"
"errors"
"log/slog"
"net/url"
@@ -84,13 +85,25 @@ type Config struct {
AsyncAdmissionMicrobatchSize int
AsyncAdmissionDispatcherEnabled bool
AsyncAdmissionDispatcherConfigured bool
RoutingMode string
ExecutionPoolID string
ExecutionPoolLabels string
WorkerID string
WorkerAdvertiseEndpoint string
WorkerOrchestratorInstanceRef string
WorkerEndpointAllowedSuffixes string
WorkerEndpointAllowPrivate bool
WorkerExecutionSecret string
WorkerExecutionCAFile string
WorkerExecutionCertFile string
WorkerExecutionKeyFile string
RouteProbeEnabled bool
RouteProbeHotIntervalSeconds int
RouteProbeColdIntervalSeconds int
RouteProbeTimeoutMS int
WorkerAutoscalingEnabled bool
WorkerReplicasNingbo int
WorkerReplicasHongkong int
WorkerMinReplicasNingbo int
WorkerMinReplicasHongkong int
WorkerMaxReplicasNingbo int
WorkerMaxReplicasHongkong int
CapacityOrchestratorAdapter string
CapacityPoolsJSON string
WorkerTargetOutstandingPerReplica int
WorkerScaleUpWindowSeconds int
WorkerScaleDownStabilizationSeconds int
@@ -101,10 +114,6 @@ type Config struct {
PostgresConnectionBudget int
PostgresNonWorkerConnectionBudget int
WorkerDatabaseMaxConns int
CapacityControllerNamespace string
CapacityControllerAPIServer string
CapacityControllerTokenFile string
CapacityControllerCAFile string
}
func Load() Config {
@@ -198,13 +207,25 @@ func Load() Config {
AsyncAdmissionDispatcherConfigured: envValue(
"AI_GATEWAY_ASYNC_ADMISSION_DISPATCHER_ENABLED",
) != "",
RoutingMode: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_ROUTING_MODE", "legacy"))),
ExecutionPoolID: strings.TrimSpace(env("AI_GATEWAY_EXECUTION_POOL_ID", "legacy-default")),
ExecutionPoolLabels: strings.TrimSpace(env("AI_GATEWAY_EXECUTION_POOL_LABELS", "{}")),
WorkerID: strings.TrimSpace(env("AI_GATEWAY_WORKER_ID", "")),
WorkerAdvertiseEndpoint: workerAdvertiseEndpoint(),
WorkerOrchestratorInstanceRef: strings.TrimSpace(env("AI_GATEWAY_ORCHESTRATOR_INSTANCE_REF", "")),
WorkerEndpointAllowedSuffixes: strings.TrimSpace(env("AI_GATEWAY_WORKER_ENDPOINT_ALLOWED_SUFFIXES", "svc,cluster.local")),
WorkerEndpointAllowPrivate: env("AI_GATEWAY_WORKER_ENDPOINT_ALLOW_PRIVATE", "true") == "true",
WorkerExecutionSecret: env("AI_GATEWAY_WORKER_EXECUTION_SECRET", env("SERVER_MAIN_INTERNAL_SECRET", env("SERVER_MAIN_INTERNAL_TOKEN", ""))),
WorkerExecutionCAFile: strings.TrimSpace(env("AI_GATEWAY_WORKER_EXECUTION_CA_FILE", "")),
WorkerExecutionCertFile: strings.TrimSpace(env("AI_GATEWAY_WORKER_EXECUTION_CERT_FILE", "")),
WorkerExecutionKeyFile: strings.TrimSpace(env("AI_GATEWAY_WORKER_EXECUTION_KEY_FILE", "")),
RouteProbeEnabled: env("AI_GATEWAY_ROUTE_PROBE_ENABLED", "true") == "true",
RouteProbeHotIntervalSeconds: envIntValidated("AI_GATEWAY_ROUTE_PROBE_HOT_INTERVAL_SECONDS", 15),
RouteProbeColdIntervalSeconds: envIntValidated("AI_GATEWAY_ROUTE_PROBE_COLD_INTERVAL_SECONDS", 60),
RouteProbeTimeoutMS: envIntValidated("AI_GATEWAY_ROUTE_PROBE_TIMEOUT_MS", 3000),
WorkerAutoscalingEnabled: env("AI_GATEWAY_WORKER_AUTOSCALING_ENABLED", "false") == "true",
WorkerReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_NINGBO", 1),
WorkerReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_REPLICAS_HONGKONG", 0),
WorkerMinReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_NINGBO", 1),
WorkerMinReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MIN_REPLICAS_HONGKONG", 0),
WorkerMaxReplicasNingbo: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_NINGBO", 1),
WorkerMaxReplicasHongkong: envOptionalIntValidated("AI_GATEWAY_WORKER_MAX_REPLICAS_HONGKONG", 0),
CapacityOrchestratorAdapter: strings.ToLower(strings.TrimSpace(env("AI_GATEWAY_CAPACITY_ORCHESTRATOR_ADAPTER", "kubernetes"))),
CapacityPoolsJSON: strings.TrimSpace(env("AI_GATEWAY_CAPACITY_POOLS", "")),
WorkerTargetOutstandingPerReplica: envOptionalIntValidated("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA", 0),
WorkerScaleUpWindowSeconds: envIntValidated("AI_GATEWAY_WORKER_SCALE_UP_WINDOW_SECONDS", 20),
WorkerScaleDownStabilizationSeconds: envIntValidated(
@@ -220,11 +241,7 @@ func Load() Config {
"AI_GATEWAY_POSTGRES_NON_WORKER_CONNECTION_BUDGET",
72,
),
WorkerDatabaseMaxConns: envIntValidated("AI_GATEWAY_WORKER_DATABASE_MAX_CONNS", 32),
CapacityControllerNamespace: env("POD_NAMESPACE", env("AI_GATEWAY_CAPACITY_CONTROLLER_NAMESPACE", "easyai")),
CapacityControllerAPIServer: env("AI_GATEWAY_CAPACITY_CONTROLLER_API_SERVER", "https://kubernetes.default.svc"),
CapacityControllerTokenFile: env("AI_GATEWAY_CAPACITY_CONTROLLER_TOKEN_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/token"),
CapacityControllerCAFile: env("AI_GATEWAY_CAPACITY_CONTROLLER_CA_FILE", "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"),
WorkerDatabaseMaxConns: envIntValidated("AI_GATEWAY_WORKER_DATABASE_MAX_CONNS", 32),
}
}
@@ -308,20 +325,46 @@ func (c Config) Validate() error {
default:
return errors.New("AI_GATEWAY_WORKER_LOAD_MODE must be adaptive or legacy")
}
switch strings.ToLower(strings.TrimSpace(c.RoutingMode)) {
case "", "legacy", "shadow", "enforced":
default:
return errors.New("AI_GATEWAY_ROUTING_MODE must be legacy, shadow, or enforced")
}
if strings.TrimSpace(c.ExecutionPoolID) != "" && !validExecutionPoolID(c.ExecutionPoolID) {
return errors.New("AI_GATEWAY_EXECUTION_POOL_ID is invalid")
}
if strings.TrimSpace(c.ExecutionPoolLabels) != "" && !jsonObject(c.ExecutionPoolLabels) {
return errors.New("AI_GATEWAY_EXECUTION_POOL_LABELS must be a JSON object")
}
if c.RouteProbeHotIntervalSeconds != 0 && (c.RouteProbeHotIntervalSeconds < 5 || c.RouteProbeHotIntervalSeconds > 300) {
return errors.New("AI_GATEWAY_ROUTE_PROBE_HOT_INTERVAL_SECONDS must be between 5 and 300")
}
if c.RouteProbeColdIntervalSeconds != 0 && (c.RouteProbeColdIntervalSeconds < c.RouteProbeHotIntervalSeconds || c.RouteProbeColdIntervalSeconds > 3600) {
return errors.New("AI_GATEWAY_ROUTE_PROBE_COLD_INTERVAL_SECONDS must be between the hot interval and 3600")
}
if c.RouteProbeTimeoutMS != 0 && (c.RouteProbeTimeoutMS < 250 || c.RouteProbeTimeoutMS > 10000) {
return errors.New("AI_GATEWAY_ROUTE_PROBE_TIMEOUT_MS must be between 250 and 10000")
}
if strings.EqualFold(c.RoutingMode, "enforced") && len(c.WorkerExecutionSecret) < 32 {
return errors.New("AI_GATEWAY_WORKER_EXECUTION_SECRET must be at least 32 bytes in enforced routing mode")
}
if (c.WorkerExecutionCertFile == "") != (c.WorkerExecutionKeyFile == "") {
return errors.New("AI_GATEWAY_WORKER_EXECUTION_CERT_FILE and AI_GATEWAY_WORKER_EXECUTION_KEY_FILE must be configured together")
}
if c.AsyncAdmissionMicrobatchSize < 1 || c.AsyncAdmissionMicrobatchSize > 32 {
return errors.New("AI_GATEWAY_ASYNC_ADMISSION_MICROBATCH_SIZE must be between 1 and 32")
}
if c.WorkerReplicasNingbo < 0 || c.WorkerReplicasHongkong < 0 ||
c.WorkerMinReplicasNingbo < 0 || c.WorkerMinReplicasHongkong < 0 ||
c.WorkerMaxReplicasNingbo < c.WorkerMinReplicasNingbo ||
c.WorkerMaxReplicasHongkong < c.WorkerMinReplicasHongkong ||
c.WorkerMaxReplicasNingbo > 64 || c.WorkerMaxReplicasHongkong > 64 {
return errors.New("Worker replica configuration must be between 0 and 64 with min not greater than max")
pools, err := c.CapacityPools()
if err != nil {
return err
}
if c.WorkerAutoscalingEnabled &&
(c.WorkerReplicasNingbo < c.WorkerMinReplicasNingbo || c.WorkerReplicasNingbo > c.WorkerMaxReplicasNingbo ||
c.WorkerReplicasHongkong < c.WorkerMinReplicasHongkong || c.WorkerReplicasHongkong > c.WorkerMaxReplicasHongkong) {
return errors.New("Worker bootstrap replicas must be within the autoscaling min/max range")
if c.RunsCapacityController() && len(pools) == 0 {
return errors.New("AI_GATEWAY_CAPACITY_POOLS must contain at least one pool for the capacity-controller role")
}
switch strings.ToLower(strings.TrimSpace(c.CapacityOrchestratorAdapter)) {
case "", "kubernetes", "static":
default:
return errors.New("AI_GATEWAY_CAPACITY_ORCHESTRATOR_ADAPTER must be kubernetes or static")
}
if c.WorkerTargetOutstandingPerReplica < 0 || c.WorkerTargetOutstandingPerReplica > 100000 {
return errors.New("AI_GATEWAY_WORKER_TARGET_OUTSTANDING_PER_REPLICA must be between 0 and 100000")
@@ -503,11 +546,79 @@ func (c Config) RunsCapacityController() bool {
return c.EffectiveProcessRole() == "capacity-controller"
}
func (c Config) RunsRouteProber() bool {
return c.RunsAsyncExecutionWorker() && c.RouteProbeEnabled && !strings.EqualFold(c.RoutingMode, "legacy")
}
func workerAdvertiseEndpoint() string {
return strings.TrimRight(strings.TrimSpace(os.Getenv("AI_GATEWAY_WORKER_ADVERTISE_ENDPOINT")), "/")
}
func validExecutionPoolID(value string) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > 128 {
return false
}
for index, character := range value {
valid := (character >= 'a' && character <= 'z') ||
(character >= 'A' && character <= 'Z') ||
(character >= '0' && character <= '9') ||
character == '.' || character == '_' || character == ':' || character == '/' || character == '-'
if !valid || (index == 0 && !((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || (character >= '0' && character <= '9'))) {
return false
}
}
return true
}
func jsonObject(value string) bool {
var object map[string]any
return json.Unmarshal([]byte(value), &object) == nil && object != nil
}
type GlobalHTTPProxyStatus struct {
HTTPProxy string
Source string
}
type ExecutionPoolCapacityConfig struct {
ID string `json:"id"`
AdapterRef string `json:"adapterRef"`
BootstrapReplicas int `json:"bootstrapReplicas"`
MinReplicas int `json:"minReplicas"`
MaxReplicas int `json:"maxReplicas"`
}
func (c Config) CapacityPools() ([]ExecutionPoolCapacityConfig, error) {
raw := strings.TrimSpace(c.CapacityPoolsJSON)
if raw == "" {
return nil, nil
}
var pools []ExecutionPoolCapacityConfig
if err := json.Unmarshal([]byte(raw), &pools); err != nil {
return nil, errors.New("AI_GATEWAY_CAPACITY_POOLS must be a JSON array")
}
if len(pools) == 0 {
return nil, errors.New("AI_GATEWAY_CAPACITY_POOLS must contain at least one pool")
}
seen := make(map[string]struct{}, len(pools))
for index := range pools {
pools[index].ID = strings.TrimSpace(pools[index].ID)
pools[index].AdapterRef = strings.TrimSpace(pools[index].AdapterRef)
if pools[index].AdapterRef == "" {
pools[index].AdapterRef = pools[index].ID
}
if !validExecutionPoolID(pools[index].ID) || pools[index].MinReplicas < 0 || pools[index].MaxReplicas < pools[index].MinReplicas || pools[index].MaxReplicas > 64 || pools[index].BootstrapReplicas < pools[index].MinReplicas || pools[index].BootstrapReplicas > pools[index].MaxReplicas {
return nil, errors.New("AI_GATEWAY_CAPACITY_POOLS contains an invalid pool definition")
}
if _, duplicate := seen[pools[index].ID]; duplicate {
return nil, errors.New("AI_GATEWAY_CAPACITY_POOLS contains duplicate pool IDs")
}
seen[pools[index].ID] = struct{}{}
}
return pools, nil
}
func LoadGlobalHTTPProxyStatus() GlobalHTTPProxyStatus {
for _, key := range []string{
"AI_GATEWAY_GLOBAL_HTTP_PROXY",