feat(worker): 实现集群限流与自适应负载

保留平台模型 RPM、TPM 和并发策略语义,增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载,避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标,并扩展本地 acceptance runner,覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证:go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过;48/48 成功,无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
2026-08-03 00:13:46 +08:00
parent 9a01fd4657
commit c28bf74230
52 changed files with 3700 additions and 272 deletions
+131
View File
@@ -0,0 +1,131 @@
package workerload
import (
"errors"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
)
type SystemSampler struct {
mu sync.Mutex
root string
lastCPUUsage int64
lastCPUTime time.Time
lastThrottled int64
}
func NewSystemSampler() *SystemSampler {
return &SystemSampler{root: "/sys/fs/cgroup"}
}
func NewSystemSamplerAt(root string) *SystemSampler {
return &SystemSampler{root: strings.TrimRight(root, "/")}
}
func (s *SystemSampler) Sample(databaseConnections, databaseMax int32) ResourceSample {
s.mu.Lock()
defer s.mu.Unlock()
now := time.Now()
memoryCurrent, memoryLimit := s.memory()
cpuUsage, throttledCount := s.cpuStat()
cpuLimit := s.cpuLimit()
cpuUtilization := 0.0
if s.lastCPUUsage > 0 && cpuUsage >= s.lastCPUUsage && !s.lastCPUTime.IsZero() {
elapsed := now.Sub(s.lastCPUTime).Seconds()
if elapsed > 0 {
cpuUtilization = float64(cpuUsage-s.lastCPUUsage) / 1_000_000 / elapsed / cpuLimit
}
}
throttled := s.lastThrottled > 0 && throttledCount > s.lastThrottled
s.lastCPUUsage = cpuUsage
s.lastThrottled = throttledCount
s.lastCPUTime = now
return ResourceSample{
MemoryCurrentBytes: memoryCurrent,
MemoryLimitBytes: memoryLimit,
CPUUtilization: bounded(cpuUtilization),
CPUThrottled: throttled,
DBConnections: databaseConnections,
DBMaxConnections: databaseMax,
SampledAt: now,
}
}
func (s *SystemSampler) memory() (int64, int64) {
current, currentErr := readIntFile(s.root + "/memory.current")
limit, limitErr := readLimitFile(s.root + "/memory.max")
if currentErr == nil && limitErr == nil {
return current, limit
}
current, _ = readIntFile(s.root + "/memory/memory.usage_in_bytes")
limit, _ = readLimitFile(s.root + "/memory/memory.limit_in_bytes")
return current, limit
}
func (s *SystemSampler) cpuStat() (usageUsec, throttled int64) {
data, err := os.ReadFile(s.root + "/cpu.stat")
if err != nil {
return 0, 0
}
for _, line := range strings.Split(string(data), "\n") {
fields := strings.Fields(line)
if len(fields) != 2 {
continue
}
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
if parseErr != nil {
continue
}
switch fields[0] {
case "usage_usec":
usageUsec = value
case "nr_throttled":
throttled = value
}
}
return usageUsec, throttled
}
func (s *SystemSampler) cpuLimit() float64 {
data, err := os.ReadFile(s.root + "/cpu.max")
if err == nil {
fields := strings.Fields(string(data))
if len(fields) == 2 && fields[0] != "max" {
quota, quotaErr := strconv.ParseFloat(fields[0], 64)
period, periodErr := strconv.ParseFloat(fields[1], 64)
if quotaErr == nil && periodErr == nil && quota > 0 && period > 0 {
return max(quota/period, .001)
}
}
}
return max(float64(runtime.GOMAXPROCS(0)), 1)
}
func readIntFile(path string) (int64, error) {
data, err := os.ReadFile(path)
if err != nil {
return 0, err
}
return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
}
func readLimitFile(path string) (int64, error) {
data, err := os.ReadFile(path)
if err != nil {
return 0, err
}
value := strings.TrimSpace(string(data))
if value == "" || value == "max" {
return 0, errors.New("cgroup limit is unlimited")
}
limit, err := strconv.ParseInt(value, 10, 64)
if err != nil || limit <= 0 || limit > 1<<60 {
return 0, errors.New("cgroup limit is not finite")
}
return limit, nil
}