fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源,负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像,并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID,报告使用独占或原子写入,负载错误记录具体阶段;数据库鉴权不可用返回带 Retry-After 的 503,避免基础设施故障被误报为 401。\n\n验证:Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
This commit is contained in:
2026-07-31 23:07:19 +08:00
parent 015ff8ea6c
commit a95184b5b6
20 changed files with 970 additions and 85 deletions
+70 -12
View File
@@ -119,6 +119,12 @@ func run(ctx context.Context, opts options) error {
return err
}
defer database.Close()
if err := verifyLocalClusterMarker(ctx, database, opts.clusterID); err != nil {
return err
}
if err := resetPreviousLocalRun(ctx, database, opts); err != nil {
return err
}
if err := verifyLocalDatabase(ctx, database, opts.clusterID); err != nil {
return err
}
@@ -212,7 +218,15 @@ func run(ctx context.Context, opts options) error {
if err != nil {
return err
}
if err := os.WriteFile(outputPath, append(payload, '\n'), 0o600); err != nil {
outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return err
}
if _, err := outputFile.Write(append(payload, '\n')); err != nil {
_ = outputFile.Close()
return err
}
if err := outputFile.Close(); err != nil {
return err
}
fmt.Printf(
@@ -226,7 +240,60 @@ func run(ctx context.Context, opts options) error {
return nil
}
func resetPreviousLocalRun(ctx context.Context, database *store.Store, opts options) error {
mode, err := database.GetGatewayTrafficMode(ctx)
if err != nil {
return err
}
if mode.Mode == "live" {
return nil
}
if mode.Mode != "validation" || strings.TrimSpace(mode.RunID) == "" {
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
}
run, err := database.GetAcceptanceRun(ctx, mode.RunID)
if err != nil {
return err
}
if run.Config["localClusterId"] != opts.clusterID {
return errors.New("refusing to replace an acceptance run owned by another cluster")
}
var outstanding int
if err := database.Pool().QueryRow(ctx, `
SELECT count(*)
FROM gateway_tasks
WHERE acceptance_run_id = $1::uuid
AND status IN ('queued','running')`, run.ID).Scan(&outstanding); err != nil {
return err
}
if outstanding != 0 {
return fmt.Errorf("refusing to replace local acceptance run with %d outstanding tasks", outstanding)
}
_, err = database.AbortAcceptanceRun(ctx, store.PromoteAcceptanceRunInput{
RunID: mode.RunID,
Revision: mode.Revision,
ReleaseSHA: mode.ReleaseSHA,
APIImageDigest: mode.APIImageDigest,
WorkerImageDigest: mode.WorkerImageDigest,
})
return err
}
func verifyLocalDatabase(ctx context.Context, database *store.Store, clusterID string) error {
if err := verifyLocalClusterMarker(ctx, database, clusterID); err != nil {
return err
}
mode, err := database.GetGatewayTrafficMode(ctx)
if err != nil {
return err
}
if mode.Mode != "live" {
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
}
return nil
}
func verifyLocalClusterMarker(ctx context.Context, database *store.Store, clusterID string) error {
var marker string
err := database.Pool().QueryRow(ctx, `
SELECT COALESCE(value->>'clusterId', '')
@@ -238,13 +305,6 @@ WHERE setting_key = $1`, acceptancesnapshot.LocalClusterSettingKey).Scan(&marker
if marker != clusterID {
return errors.New("refusing bootstrap: local cluster marker mismatch")
}
mode, err := database.GetGatewayTrafficMode(ctx)
if err != nil {
return err
}
if mode.Mode != "live" {
return fmt.Errorf("refusing bootstrap while traffic mode is %s", mode.Mode)
}
return nil
}
@@ -486,10 +546,8 @@ func privateOutputPath(path string) (string, error) {
if err != nil {
return "", err
}
if info, err := os.Lstat(absolute); err == nil {
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
return "", errors.New("runtime output must be a regular non-symlink file")
}
if _, err := os.Lstat(absolute); err == nil {
return "", errors.New("runtime output already exists")
} else if !os.IsNotExist(err) {
return "", err
}