feat(worker): 实现集群限流与自适应负载
保留平台模型 RPM、TPM 和并发策略语义，增加 PostgreSQL 集群级租约、饱和候选重选和多平台自动负载，避免突发任务固定等待首个平台。\n\n新增 Worker 实时负载采样、自适应 active/heavy 容量、心跳与管理端指标，并扩展本地 acceptance runner，覆盖三 Worker、同模型三平台 2/4/6 并发和 48 个带图视频突发任务。\n\n验证：go test ./...、go vet ./...、PostgreSQL 跨 Store 集成测试、gofmt、bash -n、ShellCheck 及本地集群 provider-burst 验收通过；48/48 成功，无越限、重复提交、重复计费、重复回调或终态资源泄漏。
This commit is contained in:
@@ -0,0 +1,381 @@
|
||||
package workerload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Phase string
|
||||
|
||||
const (
|
||||
PhasePreparing Phase = "preparing"
|
||||
PhaseWaitingUpstream Phase = "waiting_upstream"
|
||||
PhaseFinalizing Phase = "finalizing"
|
||||
)
|
||||
|
||||
type PressureState string
|
||||
|
||||
const (
|
||||
PressureNormal PressureState = "normal"
|
||||
PressureBusy PressureState = "busy"
|
||||
PressureCritical PressureState = "critical"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeAdaptive = "adaptive"
|
||||
ModeLegacy = "legacy"
|
||||
)
|
||||
|
||||
var ErrReleased = errors.New("worker load lease already released")
|
||||
|
||||
type Config struct {
|
||||
Mode string
|
||||
HardLimit int
|
||||
InitialActive int
|
||||
InitialHeavy int
|
||||
HealthySamples int
|
||||
}
|
||||
|
||||
type ResourceSample struct {
|
||||
MemoryCurrentBytes int64
|
||||
MemoryLimitBytes int64
|
||||
CPUUtilization float64
|
||||
CPUThrottled bool
|
||||
DBConnections int32
|
||||
DBMaxConnections int32
|
||||
SampledAt time.Time
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
Mode string
|
||||
ActiveLimit int
|
||||
HeavyLimit int
|
||||
ClaimLimit int
|
||||
SafeCapacity int
|
||||
ActiveTasks int
|
||||
PreparingTasks int
|
||||
WaitingUpstreamTasks int
|
||||
FinalizingTasks int
|
||||
PressureState PressureState
|
||||
PressureReason string
|
||||
MemoryUtilization float64
|
||||
CPUUtilization float64
|
||||
DBUtilization float64
|
||||
SampledAt time.Time
|
||||
}
|
||||
|
||||
type Controller struct {
|
||||
mu sync.Mutex
|
||||
|
||||
mode string
|
||||
hardLimit int
|
||||
activeLimit int
|
||||
heavyLimit int
|
||||
claimLimit int
|
||||
healthySamples int
|
||||
healthyCount int
|
||||
|
||||
preparing int
|
||||
waiting int
|
||||
finalizing int
|
||||
|
||||
last Snapshot
|
||||
wake chan struct{}
|
||||
}
|
||||
|
||||
type Lease struct {
|
||||
controller *Controller
|
||||
phase Phase
|
||||
released bool
|
||||
}
|
||||
|
||||
func New(config Config) *Controller {
|
||||
mode := strings.ToLower(strings.TrimSpace(config.Mode))
|
||||
if mode != ModeLegacy {
|
||||
mode = ModeAdaptive
|
||||
}
|
||||
if config.HardLimit < 1 {
|
||||
config.HardLimit = 1
|
||||
}
|
||||
if config.InitialActive < 1 {
|
||||
config.InitialActive = 4
|
||||
}
|
||||
if config.InitialHeavy < 1 {
|
||||
config.InitialHeavy = 1
|
||||
}
|
||||
if config.HealthySamples < 1 {
|
||||
config.HealthySamples = 3
|
||||
}
|
||||
active := min(config.InitialActive, config.HardLimit)
|
||||
heavy := min(config.InitialHeavy, active)
|
||||
if mode == ModeLegacy {
|
||||
active = config.HardLimit
|
||||
heavy = config.HardLimit
|
||||
}
|
||||
controller := &Controller{
|
||||
mode: mode, hardLimit: config.HardLimit,
|
||||
activeLimit: active, heavyLimit: heavy, claimLimit: active,
|
||||
healthySamples: config.HealthySamples,
|
||||
wake: make(chan struct{}, 1),
|
||||
}
|
||||
controller.last = controller.snapshotLocked(ResourceSample{SampledAt: time.Now()})
|
||||
return controller
|
||||
}
|
||||
|
||||
func (c *Controller) Observe(sample ResourceSample) Snapshot {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if sample.SampledAt.IsZero() {
|
||||
sample.SampledAt = time.Now()
|
||||
}
|
||||
memory := utilization(sample.MemoryCurrentBytes, sample.MemoryLimitBytes)
|
||||
database := utilization(int64(sample.DBConnections), int64(sample.DBMaxConnections))
|
||||
cpu := bounded(sample.CPUUtilization)
|
||||
state, reason := pressure(memory, cpu, database, sample.CPUThrottled)
|
||||
if c.mode == ModeLegacy {
|
||||
c.activeLimit = c.hardLimit
|
||||
c.heavyLimit = c.hardLimit
|
||||
state = PressureNormal
|
||||
reason = "legacy"
|
||||
} else {
|
||||
c.adjustLocked(state, memory, cpu, database)
|
||||
}
|
||||
c.last = c.snapshotLocked(sample)
|
||||
c.last.PressureState = state
|
||||
c.last.PressureReason = reason
|
||||
c.last.MemoryUtilization = memory
|
||||
c.last.CPUUtilization = cpu
|
||||
c.last.DBUtilization = database
|
||||
c.last.SafeCapacity = c.activeLimit
|
||||
if state == PressureCritical {
|
||||
c.last.SafeCapacity = 0
|
||||
}
|
||||
c.signalLocked()
|
||||
return c.last
|
||||
}
|
||||
|
||||
func (c *Controller) SetClaimLimit(limit int) Snapshot {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if limit < 0 {
|
||||
limit = 0
|
||||
}
|
||||
c.claimLimit = min(limit, c.hardLimit)
|
||||
c.last = c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
c.signalLocked()
|
||||
return c.last
|
||||
}
|
||||
|
||||
func (c *Controller) Snapshot() Snapshot {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
}
|
||||
|
||||
func (c *Controller) TryStart() (*Lease, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
activeLimit := min(c.activeLimit, c.claimLimit)
|
||||
if activeLimit <= 0 || c.activeLocked() >= activeLimit || c.preparing+c.finalizing >= c.heavyLimit {
|
||||
return nil, false
|
||||
}
|
||||
c.preparing++
|
||||
c.last = c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
return &Lease{controller: c, phase: PhasePreparing}, true
|
||||
}
|
||||
|
||||
func (l *Lease) EnterWaiting() error {
|
||||
if l == nil || l.controller == nil {
|
||||
return nil
|
||||
}
|
||||
c := l.controller
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if l.released {
|
||||
return ErrReleased
|
||||
}
|
||||
if l.phase == PhaseWaitingUpstream {
|
||||
return nil
|
||||
}
|
||||
c.decrementPhaseLocked(l.phase)
|
||||
c.waiting++
|
||||
l.phase = PhaseWaitingUpstream
|
||||
c.last = c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
c.signalLocked()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *Lease) EnterFinalizing(ctx context.Context) error {
|
||||
if l == nil || l.controller == nil {
|
||||
return nil
|
||||
}
|
||||
c := l.controller
|
||||
for {
|
||||
c.mu.Lock()
|
||||
if l.released {
|
||||
c.mu.Unlock()
|
||||
return ErrReleased
|
||||
}
|
||||
if l.phase == PhaseFinalizing {
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
// Even under critical pressure, let one submitted task at a time finish
|
||||
// and release its provider lease instead of deadlocking the drain path.
|
||||
heavyLimit := max(c.heavyLimit, 1)
|
||||
if c.preparing+c.finalizing < heavyLimit {
|
||||
c.decrementPhaseLocked(l.phase)
|
||||
c.finalizing++
|
||||
l.phase = PhaseFinalizing
|
||||
c.last = c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
wake := c.wake
|
||||
c.mu.Unlock()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-wake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Lease) Release() {
|
||||
if l == nil || l.controller == nil {
|
||||
return
|
||||
}
|
||||
c := l.controller
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if l.released {
|
||||
return
|
||||
}
|
||||
c.decrementPhaseLocked(l.phase)
|
||||
l.released = true
|
||||
c.last = c.snapshotLocked(ResourceSample{SampledAt: c.last.SampledAt})
|
||||
c.signalLocked()
|
||||
}
|
||||
|
||||
func (l *Lease) Phase() Phase {
|
||||
if l == nil || l.controller == nil {
|
||||
return ""
|
||||
}
|
||||
l.controller.mu.Lock()
|
||||
defer l.controller.mu.Unlock()
|
||||
return l.phase
|
||||
}
|
||||
|
||||
func (c *Controller) adjustLocked(state PressureState, memory, cpu, database float64) {
|
||||
switch state {
|
||||
case PressureCritical:
|
||||
c.healthyCount = 0
|
||||
c.activeLimit = max(1, c.activeLimit/2)
|
||||
c.heavyLimit = 1
|
||||
case PressureBusy:
|
||||
c.healthyCount = 0
|
||||
step := max(1, c.activeLimit/4)
|
||||
c.activeLimit = max(1, c.activeLimit-step)
|
||||
c.heavyLimit = min(c.heavyLimit, max(1, (c.activeLimit+3)/4))
|
||||
default:
|
||||
if memory > .60 || cpu > .65 || database > .65 || c.activeLocked() < min(c.activeLimit, c.claimLimit) {
|
||||
c.healthyCount = 0
|
||||
return
|
||||
}
|
||||
c.healthyCount++
|
||||
if c.healthyCount < c.healthySamples {
|
||||
return
|
||||
}
|
||||
c.healthyCount = 0
|
||||
step := max(1, c.activeLimit/4)
|
||||
c.activeLimit = min(c.hardLimit, c.activeLimit+step)
|
||||
c.heavyLimit = min(c.activeLimit, max(1, (c.activeLimit+3)/4))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) snapshotLocked(sample ResourceSample) Snapshot {
|
||||
sampledAt := sample.SampledAt
|
||||
if sampledAt.IsZero() {
|
||||
sampledAt = c.last.SampledAt
|
||||
}
|
||||
safeCapacity := c.activeLimit
|
||||
if c.last.PressureState == PressureCritical {
|
||||
safeCapacity = 0
|
||||
}
|
||||
return Snapshot{
|
||||
Mode: c.mode,
|
||||
ActiveLimit: c.activeLimit,
|
||||
HeavyLimit: c.heavyLimit,
|
||||
ClaimLimit: c.claimLimit,
|
||||
SafeCapacity: safeCapacity,
|
||||
ActiveTasks: c.activeLocked(),
|
||||
PreparingTasks: c.preparing,
|
||||
WaitingUpstreamTasks: c.waiting,
|
||||
FinalizingTasks: c.finalizing,
|
||||
PressureState: c.last.PressureState,
|
||||
PressureReason: c.last.PressureReason,
|
||||
MemoryUtilization: c.last.MemoryUtilization,
|
||||
CPUUtilization: c.last.CPUUtilization,
|
||||
DBUtilization: c.last.DBUtilization,
|
||||
SampledAt: sampledAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) activeLocked() int { return c.preparing + c.waiting + c.finalizing }
|
||||
|
||||
func (c *Controller) decrementPhaseLocked(phase Phase) {
|
||||
switch phase {
|
||||
case PhasePreparing:
|
||||
c.preparing = max(0, c.preparing-1)
|
||||
case PhaseWaitingUpstream:
|
||||
c.waiting = max(0, c.waiting-1)
|
||||
case PhaseFinalizing:
|
||||
c.finalizing = max(0, c.finalizing-1)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) signalLocked() {
|
||||
select {
|
||||
case c.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func pressure(memory, cpu, database float64, throttled bool) (PressureState, string) {
|
||||
switch {
|
||||
case memory >= .90:
|
||||
return PressureCritical, "memory"
|
||||
case database >= .90:
|
||||
return PressureCritical, "database"
|
||||
case cpu >= .95 && throttled:
|
||||
return PressureCritical, "cpu_throttled"
|
||||
case memory >= .75:
|
||||
return PressureBusy, "memory"
|
||||
case database >= .80:
|
||||
return PressureBusy, "database"
|
||||
case cpu >= .80 || throttled:
|
||||
return PressureBusy, "cpu"
|
||||
default:
|
||||
return PressureNormal, ""
|
||||
}
|
||||
}
|
||||
|
||||
func utilization(current, limit int64) float64 {
|
||||
if current <= 0 || limit <= 0 {
|
||||
return 0
|
||||
}
|
||||
return bounded(float64(current) / float64(limit))
|
||||
}
|
||||
|
||||
func bounded(value float64) float64 {
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 1 {
|
||||
return 1
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package workerload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestControllerStartsConservativelyAndGrowsUnderSustainedDemand(t *testing.T) {
|
||||
controller := New(Config{Mode: ModeAdaptive, HardLimit: 16, InitialActive: 4, InitialHeavy: 1, HealthySamples: 2})
|
||||
leases := make([]*Lease, 0, 4)
|
||||
for range 4 {
|
||||
lease, ok := controller.TryStart()
|
||||
if !ok {
|
||||
t.Fatal("initial task was not admitted")
|
||||
}
|
||||
_ = lease.EnterWaiting()
|
||||
leases = append(leases, lease)
|
||||
}
|
||||
for range 2 {
|
||||
controller.Observe(ResourceSample{MemoryCurrentBytes: 40, MemoryLimitBytes: 100, CPUUtilization: .4, DBConnections: 4, DBMaxConnections: 20})
|
||||
}
|
||||
snapshot := controller.Snapshot()
|
||||
if snapshot.ActiveLimit != 5 || snapshot.HeavyLimit != 2 {
|
||||
t.Fatalf("grown snapshot=%+v, want active=5 heavy=2", snapshot)
|
||||
}
|
||||
for _, lease := range leases {
|
||||
lease.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func TestControllerBusyAndCriticalPressureReduceNewClaims(t *testing.T) {
|
||||
controller := New(Config{Mode: ModeAdaptive, HardLimit: 16, InitialActive: 8, InitialHeavy: 2})
|
||||
busy := controller.Observe(ResourceSample{MemoryCurrentBytes: 80, MemoryLimitBytes: 100})
|
||||
if busy.PressureState != PressureBusy || busy.SafeCapacity >= 8 {
|
||||
t.Fatalf("busy snapshot=%+v", busy)
|
||||
}
|
||||
critical := controller.Observe(ResourceSample{MemoryCurrentBytes: 95, MemoryLimitBytes: 100})
|
||||
if critical.PressureState != PressureCritical || critical.SafeCapacity != 0 {
|
||||
t.Fatalf("critical snapshot=%+v", critical)
|
||||
}
|
||||
controller.SetClaimLimit(0)
|
||||
if _, ok := controller.TryStart(); ok {
|
||||
t.Fatal("critical controller admitted a new task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitingReleasesHeavyPermitAndFinalizingReacquiresIt(t *testing.T) {
|
||||
controller := New(Config{Mode: ModeAdaptive, HardLimit: 4, InitialActive: 4, InitialHeavy: 1})
|
||||
first, ok := controller.TryStart()
|
||||
if !ok {
|
||||
t.Fatal("first lease unavailable")
|
||||
}
|
||||
if _, ok := controller.TryStart(); ok {
|
||||
t.Fatal("second preparing task bypassed heavy limit")
|
||||
}
|
||||
if err := first.EnterWaiting(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, ok := controller.TryStart()
|
||||
if !ok {
|
||||
t.Fatal("waiting task did not release heavy permit")
|
||||
}
|
||||
if err := second.EnterWaiting(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := first.EnterFinalizing(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocked := make(chan error, 1)
|
||||
go func() { blocked <- second.EnterFinalizing(ctx) }()
|
||||
select {
|
||||
case err := <-blocked:
|
||||
t.Fatalf("second finalizer did not wait: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
first.Release()
|
||||
if err := <-blocked; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second.Release()
|
||||
if snapshot := controller.Snapshot(); snapshot.ActiveTasks != 0 {
|
||||
t.Fatalf("active tasks=%d, want 0", snapshot.ActiveTasks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyModeUsesHardLimit(t *testing.T) {
|
||||
controller := New(Config{Mode: ModeLegacy, HardLimit: 7})
|
||||
snapshot := controller.Observe(ResourceSample{MemoryCurrentBytes: 99, MemoryLimitBytes: 100, CPUUtilization: 1, CPUThrottled: true})
|
||||
if snapshot.ActiveLimit != 7 || snapshot.HeavyLimit != 7 || snapshot.SafeCapacity != 7 || snapshot.PressureReason != "legacy" {
|
||||
t.Fatalf("legacy snapshot=%+v", snapshot)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package workerload
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type SystemSampler struct {
|
||||
mu sync.Mutex
|
||||
|
||||
root string
|
||||
lastCPUUsage int64
|
||||
lastCPUTime time.Time
|
||||
lastThrottled int64
|
||||
}
|
||||
|
||||
func NewSystemSampler() *SystemSampler {
|
||||
return &SystemSampler{root: "/sys/fs/cgroup"}
|
||||
}
|
||||
|
||||
func NewSystemSamplerAt(root string) *SystemSampler {
|
||||
return &SystemSampler{root: strings.TrimRight(root, "/")}
|
||||
}
|
||||
|
||||
func (s *SystemSampler) Sample(databaseConnections, databaseMax int32) ResourceSample {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
now := time.Now()
|
||||
memoryCurrent, memoryLimit := s.memory()
|
||||
cpuUsage, throttledCount := s.cpuStat()
|
||||
cpuLimit := s.cpuLimit()
|
||||
cpuUtilization := 0.0
|
||||
if s.lastCPUUsage > 0 && cpuUsage >= s.lastCPUUsage && !s.lastCPUTime.IsZero() {
|
||||
elapsed := now.Sub(s.lastCPUTime).Seconds()
|
||||
if elapsed > 0 {
|
||||
cpuUtilization = float64(cpuUsage-s.lastCPUUsage) / 1_000_000 / elapsed / cpuLimit
|
||||
}
|
||||
}
|
||||
throttled := s.lastThrottled > 0 && throttledCount > s.lastThrottled
|
||||
s.lastCPUUsage = cpuUsage
|
||||
s.lastThrottled = throttledCount
|
||||
s.lastCPUTime = now
|
||||
return ResourceSample{
|
||||
MemoryCurrentBytes: memoryCurrent,
|
||||
MemoryLimitBytes: memoryLimit,
|
||||
CPUUtilization: bounded(cpuUtilization),
|
||||
CPUThrottled: throttled,
|
||||
DBConnections: databaseConnections,
|
||||
DBMaxConnections: databaseMax,
|
||||
SampledAt: now,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SystemSampler) memory() (int64, int64) {
|
||||
current, currentErr := readIntFile(s.root + "/memory.current")
|
||||
limit, limitErr := readLimitFile(s.root + "/memory.max")
|
||||
if currentErr == nil && limitErr == nil {
|
||||
return current, limit
|
||||
}
|
||||
current, _ = readIntFile(s.root + "/memory/memory.usage_in_bytes")
|
||||
limit, _ = readLimitFile(s.root + "/memory/memory.limit_in_bytes")
|
||||
return current, limit
|
||||
}
|
||||
|
||||
func (s *SystemSampler) cpuStat() (usageUsec, throttled int64) {
|
||||
data, err := os.ReadFile(s.root + "/cpu.stat")
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
continue
|
||||
}
|
||||
value, parseErr := strconv.ParseInt(fields[1], 10, 64)
|
||||
if parseErr != nil {
|
||||
continue
|
||||
}
|
||||
switch fields[0] {
|
||||
case "usage_usec":
|
||||
usageUsec = value
|
||||
case "nr_throttled":
|
||||
throttled = value
|
||||
}
|
||||
}
|
||||
return usageUsec, throttled
|
||||
}
|
||||
|
||||
func (s *SystemSampler) cpuLimit() float64 {
|
||||
data, err := os.ReadFile(s.root + "/cpu.max")
|
||||
if err == nil {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) == 2 && fields[0] != "max" {
|
||||
quota, quotaErr := strconv.ParseFloat(fields[0], 64)
|
||||
period, periodErr := strconv.ParseFloat(fields[1], 64)
|
||||
if quotaErr == nil && periodErr == nil && quota > 0 && period > 0 {
|
||||
return max(quota/period, .001)
|
||||
}
|
||||
}
|
||||
}
|
||||
return max(float64(runtime.GOMAXPROCS(0)), 1)
|
||||
}
|
||||
|
||||
func readIntFile(path string) (int64, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64)
|
||||
}
|
||||
|
||||
func readLimitFile(path string) (int64, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value := strings.TrimSpace(string(data))
|
||||
if value == "" || value == "max" {
|
||||
return 0, errors.New("cgroup limit is unlimited")
|
||||
}
|
||||
limit, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || limit <= 0 || limit > 1<<60 {
|
||||
return 0, errors.New("cgroup limit is not finite")
|
||||
}
|
||||
return limit, nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package workerload
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSystemSamplerReadsCgroupV2(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeSamplerFixture(t, filepath.Join(root, "memory.current"), "50\n")
|
||||
writeSamplerFixture(t, filepath.Join(root, "memory.max"), "100\n")
|
||||
writeSamplerFixture(t, filepath.Join(root, "cpu.max"), "100000 100000\n")
|
||||
writeSamplerFixture(t, filepath.Join(root, "cpu.stat"), "usage_usec 100000\nnr_throttled 1\n")
|
||||
sampler := NewSystemSamplerAt(root)
|
||||
first := sampler.Sample(2, 10)
|
||||
if first.MemoryCurrentBytes != 50 || first.MemoryLimitBytes != 100 || first.DBConnections != 2 {
|
||||
t.Fatalf("first sample=%+v", first)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
writeSamplerFixture(t, filepath.Join(root, "cpu.stat"), "usage_usec 105000\nnr_throttled 2\n")
|
||||
second := sampler.Sample(3, 10)
|
||||
if second.CPUUtilization <= 0 || !second.CPUThrottled {
|
||||
t.Fatalf("second sample=%+v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func writeSamplerFixture(t *testing.T, path, value string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(value), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user