fix(acceptance): 隔离控制面抖动与租约瞬态故障

线上 P24 验收暴露出高频 kubectl exec 放大 K3s API 压力、门禁查询挤占关键连接池,以及 PostgreSQL 锁超时被误判为租约所有权丢失。

本次合并验收身份查询、在租约有效期内重试瞬态续期错误、修复人工审核残留 attempt,并增加滚动后 etcd 稳定窗口、节点直连指标和双站独立报告。

验证:Go 全量测试、go vet、聚焦 race、gofmt、迁移安全检查、bash -n、ShellCheck、manual release test。
This commit is contained in:
2026-08-01 01:51:44 +08:00
parent be6ce7f78a
commit 132cda35d8
10 changed files with 495 additions and 56 deletions
+32 -12
View File
@@ -2107,32 +2107,48 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
runCtx, cancelRun := context.WithCancel(ctx)
renewCtx, cancelRenew := context.WithCancel(ctx)
done := make(chan error, 1)
minimumTTL := 120 * time.Second
for _, lease := range leases {
if lease.TTL > 0 && lease.TTL < minimumTTL {
minimumTTL = lease.TTL
}
}
go func() {
ticker := time.NewTicker(interval)
defer ticker.Stop()
timer := time.NewTimer(interval)
defer timer.Stop()
leaseDeadline := time.Now().Add(minimumTTL)
for {
select {
case <-renewCtx.Done():
done <- nil
return
case <-ticker.C:
case <-timer.C:
if err := s.coordinationStore.RenewConcurrencyLeases(renewCtx, leases); err != nil {
if renewCtx.Err() != nil {
done <- nil
return
}
outcome := "failure"
if errors.Is(err, store.ErrConcurrencyLeaseLost) {
outcome = "lost"
if concurrencyLeaseRenewalIsFatal(err, time.Now(), leaseDeadline) {
outcome := "failure"
if errors.Is(err, store.ErrConcurrencyLeaseLost) {
outcome = "lost"
}
s.observeConcurrencyLeaseRenewal(outcome)
s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
done <- err
cancelRun()
return
}
s.observeConcurrencyLeaseRenewal(outcome)
s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
done <- err
cancelRun()
return
s.observeConcurrencyLeaseRenewal("failure")
s.logger.Warn("concurrency lease renewal failed; retrying before lease expiry",
"taskID", taskID, "leaseCount", len(leases), "outcome", "transient", "error", err)
timer.Reset(leaseRenewalRetryInterval)
continue
}
s.observeConcurrencyLeaseRenewal("success")
leaseDeadline = time.Now().Add(minimumTTL)
timer.Reset(interval)
}
}
}()
@@ -2144,6 +2160,10 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
}
}
func concurrencyLeaseRenewalIsFatal(err error, now time.Time, leaseDeadline time.Time) bool {
return errors.Is(err, store.ErrConcurrencyLeaseLost) || !now.Before(leaseDeadline)
}
func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
observer, ok := s.billingMetrics.(interface {
ObserveConcurrencyLeaseRenewal(string)