fix(acceptance): 阻断本地控制面漂移污染验收

本地 K3s 节点此前在 Kubelet 中登记为宿主机资源,负载可挤压 etcd 并在 server 重启后继续污染同一 Run。现在为三节点设置真实可调度预算、独立 etcd/数据卷和锁定依赖镜像,并持续核对 server 与 API/etcd 健康。\n\n每次验收生成独立 Run ID,报告使用独占或原子写入,负载错误记录具体阶段;数据库鉴权不可用返回带 Retry-After 的 503,避免基础设施故障被误报为 401。\n\n验证:Go 全量测试、go vet、gofmt、OpenAPI、bash -n、ShellCheck、报告测试和 k3d 配置解析均通过。
This commit is contained in:
2026-07-31 23:07:19 +08:00
parent 015ff8ea6c
commit a95184b5b6
20 changed files with 970 additions and 85 deletions
+58 -19
View File
@@ -59,15 +59,16 @@ type options struct {
}
type report struct {
SchemaVersion string `json:"schemaVersion"`
RunID string `json:"runId"`
Profile string `json:"profile"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
Passed bool `json:"passed"`
Phases []phaseReport `json:"phases"`
Failure string `json:"failure,omitempty"`
SecretSafe bool `json:"secretSafe"`
SchemaVersion string `json:"schemaVersion"`
RunID string `json:"runId"`
Profile string `json:"profile"`
StartedAt time.Time `json:"startedAt"`
FinishedAt time.Time `json:"finishedAt"`
Passed bool `json:"passed"`
Phases []phaseReport `json:"phases"`
Failure string `json:"failure,omitempty"`
FailureOperation string `json:"failureOperation,omitempty"`
SecretSafe bool `json:"secretSafe"`
}
type phaseReport struct {
@@ -101,6 +102,25 @@ type httpStatusError struct {
Body string
}
type operationError struct {
Operation string
Err error
}
func (e *operationError) Error() string { return e.Operation + ": " + e.Err.Error() }
func (e *operationError) Unwrap() error { return e.Err }
func withOperation(operation string, err error) error {
if err == nil {
return nil
}
var existing *operationError
if errors.As(err, &existing) {
return err
}
return &operationError{Operation: operation, Err: err}
}
func (e *httpStatusError) Error() string {
return fmt.Sprintf("HTTP %d: %s", e.Status, e.Body)
}
@@ -137,11 +157,15 @@ func main() {
result.Passed = runErr == nil
if runErr != nil {
result.Failure = redactError(runErr.Error(), opts)
var operationErr *operationError
if errors.As(runErr, &operationErr) {
result.FailureOperation = operationErr.Operation
}
}
payload, _ := json.MarshalIndent(result, "", " ")
fmt.Println(string(payload))
if opts.reportPath != "" {
if err := os.WriteFile(opts.reportPath, append(payload, '\n'), 0o600); err != nil {
if err := writeReportExclusive(opts.reportPath, payload); err != nil {
fmt.Fprintln(os.Stderr, "write acceptance report failed:", err)
os.Exit(1)
}
@@ -151,6 +175,18 @@ func main() {
}
}
func writeReportExclusive(path string, payload []byte) error {
reportFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err != nil {
return err
}
if _, err := reportFile.Write(append(payload, '\n')); err != nil {
_ = reportFile.Close()
return err
}
return reportFile.Close()
}
func parseOptions() (options, error) {
var profile string
var reportPath string
@@ -347,14 +383,15 @@ func runGemini(
response, err = client.Do(req)
if err == nil {
if response.StatusCode != http.StatusOK {
err = responseStatusError(response)
err = withOperation("gemini_submit", responseStatusError(response))
} else {
var size int64
var outputHash string
size, outputHash, err = streamGeminiImageHash(response.Body)
err = withOperation("gemini_response", err)
_ = response.Body.Close()
if err == nil && expectedOutputBytes > 0 && (size != int64(expectedOutputBytes) || outputHash != expectedHash) {
err = fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash)
err = withOperation("gemini_response", fmt.Errorf("Gemini output mismatch: bytes=%d hash=%s", size, outputHash))
}
if err == nil {
mu.Lock()
@@ -367,6 +404,7 @@ func runGemini(
}
}
}
err = withOperation("gemini_submit", err)
latency := time.Since(requestStarted)
mu.Lock()
latencies[index] = latency
@@ -503,7 +541,7 @@ func runVideo(
response, err = client.Do(req)
if err == nil {
if response.StatusCode != http.StatusAccepted {
err = responseStatusError(response)
err = withOperation("video_submit", responseStatusError(response))
} else {
var accepted struct {
TaskID string `json:"taskId"`
@@ -517,6 +555,7 @@ func runVideo(
}
}
}
err = withOperation("video_submit", err)
mu.Lock()
latencies[index] = time.Since(requestStarted)
if !realUpstream && logicalIndex%4 == 0 {
@@ -742,7 +781,7 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI
endpoint := opts.gateways[index%len(opts.gateways)] + "/api/v1/ai/result/" + url.PathEscape(taskID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return err
return withOperation("video_poll", err)
}
opts.setHeaders(req, index, realUpstream)
response, err := client.Do(req)
@@ -758,20 +797,20 @@ func pollVideoTask(ctx context.Context, client *http.Client, opts options, taskI
case "succeeded", "success":
mediaURL := findMediaURL(payload)
if mediaURL == "" {
return fmt.Errorf("video task %s succeeded without a media URL", taskID)
return withOperation("video_poll", fmt.Errorf("video task %s succeeded without a media URL", taskID))
}
return validateVideoAsset(ctx, client, opts, mediaURL, index)
return withOperation("video_download", validateVideoAsset(ctx, client, opts, mediaURL, index))
case "failed", "cancelled", "canceled":
return fmt.Errorf("video task %s finished with status %s", taskID, status)
return withOperation("video_poll", fmt.Errorf("video task %s finished with status %s", taskID, status))
}
}
}
if err != nil {
return err
return withOperation("video_poll", err)
}
select {
case <-ctx.Done():
return ctx.Err()
return withOperation("video_poll", ctx.Err())
case <-ticker.C:
}
}