fix(acceptance): 隔离控制面抖动与租约瞬态故障

线上 P24 验收暴露出高频 kubectl exec 放大 K3s API 压力、门禁查询挤占关键连接池，以及 PostgreSQL 锁超时被误判为租约所有权丢失。

本次合并验收身份查询、在租约有效期内重试瞬态续期错误、修复人工审核残留 attempt，并增加滚动后 etcd 稳定窗口、节点直连指标和双站独立报告。

验证：Go 全量测试、go vet、聚焦 race、gofmt、迁移安全检查、bash -n、ShellCheck、manual release test。

apps/api/internal/runner/execution_lease.go

@@ -11,24 +11,37 @@ import (
 const (
 	taskExecutionLeaseTTL      = 5 * time.Minute
 	taskExecutionRenewInterval = 30 * time.Second
+	leaseRenewalRetryInterval  = 2 * time.Second
 )
 
 func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.CancelFunc, taskID string, executionToken string) {
-	ticker := time.NewTicker(taskExecutionRenewInterval)
-	defer ticker.Stop()
+	timer := time.NewTimer(taskExecutionRenewInterval)
+	defer timer.Stop()
+	leaseDeadline := time.Now().Add(taskExecutionLeaseTTL)
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-timer.C:
 			if err := s.coordinationStore.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil {
 				if errors.Is(err, store.ErrTaskExecutionFinished) {
 					return
 				}
-				s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost")
-				cancel()
-				return
+				if taskExecutionLeaseRenewalIsFatal(err, time.Now(), leaseDeadline) {
+					s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost", "error", err)
+					cancel()
+					return
+				}
+				s.logger.Warn("task execution lease renewal failed; retrying before lease expiry", "taskID", taskID, "error_category", "task_execution_lease_renewal_transient", "error", err)
+				timer.Reset(leaseRenewalRetryInterval)
+				continue
 			}
+			leaseDeadline = time.Now().Add(taskExecutionLeaseTTL)
+			timer.Reset(taskExecutionRenewInterval)
 		}
 	}
 }
+
+func taskExecutionLeaseRenewalIsFatal(err error, now time.Time, leaseDeadline time.Time) bool {
+	return errors.Is(err, store.ErrTaskExecutionLeaseLost) || !now.Before(leaseDeadline)
+}

apps/api/internal/runner/execution_lease_test.go

@@ -0,0 +1,33 @@
+package runner
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
+)
+
+func TestLeaseRenewalFatalityDistinguishesTransientDatabaseErrors(t *testing.T) {
+	now := time.Now()
+	deadline := now.Add(time.Minute)
+	transient := errors.New("lock timeout")
+	if taskExecutionLeaseRenewalIsFatal(transient, now, deadline) {
+		t.Fatal("transient task lease renewal error must be retried before expiry")
+	}
+	if concurrencyLeaseRenewalIsFatal(transient, now, deadline) {
+		t.Fatal("transient concurrency lease renewal error must be retried before expiry")
+	}
+	if !taskExecutionLeaseRenewalIsFatal(store.ErrTaskExecutionLeaseLost, now, deadline) {
+		t.Fatal("confirmed task lease ownership loss must be fatal")
+	}
+	if !concurrencyLeaseRenewalIsFatal(store.ErrConcurrencyLeaseLost, now, deadline) {
+		t.Fatal("confirmed concurrency lease ownership loss must be fatal")
+	}
+	if !taskExecutionLeaseRenewalIsFatal(transient, deadline, deadline) {
+		t.Fatal("task lease renewal error at expiry must be fatal")
+	}
+	if !concurrencyLeaseRenewalIsFatal(transient, deadline, deadline) {
+		t.Fatal("concurrency lease renewal error at expiry must be fatal")
+	}
+}

apps/api/internal/runner/service.go

@@ -2107,32 +2107,48 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
 	runCtx, cancelRun := context.WithCancel(ctx)
 	renewCtx, cancelRenew := context.WithCancel(ctx)
 	done := make(chan error, 1)
+	minimumTTL := 120 * time.Second
+	for _, lease := range leases {
+		if lease.TTL > 0 && lease.TTL < minimumTTL {
+			minimumTTL = lease.TTL
+		}
+	}
 	go func() {
-		ticker := time.NewTicker(interval)
-		defer ticker.Stop()
+		timer := time.NewTimer(interval)
+		defer timer.Stop()
+		leaseDeadline := time.Now().Add(minimumTTL)
 		for {
 			select {
 			case <-renewCtx.Done():
 				done <- nil
 				return
-			case <-ticker.C:
+			case <-timer.C:
 				if err := s.coordinationStore.RenewConcurrencyLeases(renewCtx, leases); err != nil {
 					if renewCtx.Err() != nil {
 						done <- nil
 						return
 					}
-					outcome := "failure"
-					if errors.Is(err, store.ErrConcurrencyLeaseLost) {
-						outcome = "lost"
+					if concurrencyLeaseRenewalIsFatal(err, time.Now(), leaseDeadline) {
+						outcome := "failure"
+						if errors.Is(err, store.ErrConcurrencyLeaseLost) {
+							outcome = "lost"
+						}
+						s.observeConcurrencyLeaseRenewal(outcome)
+						s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
+							"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
+						done <- err
+						cancelRun()
+						return
 					}
-					s.observeConcurrencyLeaseRenewal(outcome)
-					s.logger.Error("concurrency lease renewal failed; cancelling upstream execution",
-						"taskID", taskID, "leaseCount", len(leases), "outcome", outcome, "error", err)
-					done <- err
-					cancelRun()
-					return
+					s.observeConcurrencyLeaseRenewal("failure")
+					s.logger.Warn("concurrency lease renewal failed; retrying before lease expiry",
+						"taskID", taskID, "leaseCount", len(leases), "outcome", "transient", "error", err)
+					timer.Reset(leaseRenewalRetryInterval)
+					continue
 				}
 				s.observeConcurrencyLeaseRenewal("success")
+				leaseDeadline = time.Now().Add(minimumTTL)
+				timer.Reset(interval)
 			}
 		}
 	}()
@@ -2144,6 +2160,10 @@ func (s *Service) startConcurrencyLeaseRenewal(ctx context.Context, taskID strin
 	}
 }
 
+func concurrencyLeaseRenewalIsFatal(err error, now time.Time, leaseDeadline time.Time) bool {
+	return errors.Is(err, store.ErrConcurrencyLeaseLost) || !now.Before(leaseDeadline)
+}
+
 func (s *Service) observeConcurrencyLeaseRenewal(outcome string) {
 	observer, ok := s.billingMetrics.(interface {
 		ObserveConcurrencyLeaseRenewal(string)