feat(acceptance): 增加生产同构媒体压力验收模式

引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器,覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。

新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排;Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。

验证:Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
2026-07-30 23:06:19 +08:00
parent f33d6d64e0
commit 3053ba4925
45 changed files with 5214 additions and 45 deletions
@@ -0,0 +1,300 @@
package httpapi
import (
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
)
type taskTrafficAdmission struct {
RunMode string
AcceptanceRunID string
}
type taskTrafficError struct {
Status int
Code string
Message string
Err error
}
func (e *taskTrafficError) Error() string {
return e.Message
}
func (e *taskTrafficError) ErrorCode() string {
return e.Code
}
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
runID, err := s.store.AuthorizeAcceptanceTask(
r.Context(),
r.Header.Get(acceptanceRunHeader),
r.Header.Get(acceptanceTokenHeader),
user,
)
if err != nil {
switch {
case errors.Is(err, store.ErrProductionTrafficPaused):
return taskTrafficAdmission{}, &taskTrafficError{
Status: http.StatusServiceUnavailable, Code: "validation_in_progress",
Message: "new production tasks are paused while validation is running", Err: err,
}
case errors.Is(err, store.ErrAcceptanceNotAuthorized):
return taskTrafficAdmission{}, &taskTrafficError{
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
Message: "acceptance credentials do not match the active run", Err: err,
}
case errors.Is(err, store.ErrAcceptanceRunNotActive):
return taskTrafficAdmission{}, &taskTrafficError{
Status: http.StatusConflict, Code: "acceptance_run_not_active",
Message: "acceptance headers are not valid while live traffic is enabled", Err: err,
}
default:
return taskTrafficAdmission{}, &taskTrafficError{
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
Message: "task traffic gate is unavailable", Err: err,
}
}
}
if runID != "" {
runMode := "acceptance"
if strings.EqualFold(strings.TrimSpace(r.Header.Get(acceptanceUpstreamHeader)), "real") {
runMode = "acceptance_canary"
}
return taskTrafficAdmission{RunMode: runMode, AcceptanceRunID: runID}, nil
}
return taskTrafficAdmission{RunMode: "production"}, nil
}
func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) {
if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" {
return admission.RunMode, nil
}
requested := runModeFromRequest(body)
requestedMode := strings.ToLower(strings.TrimSpace(requested))
if requestedMode == "acceptance" || requestedMode == "acceptance_canary" {
return "", &taskTrafficError{
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
Message: "acceptance mode is available only through an active acceptance run",
}
}
if strings.EqualFold(strings.TrimSpace(s.cfg.AppEnv), "production") && requestedMode == "simulation" {
return "", &taskTrafficError{
Status: http.StatusForbidden, Code: "simulation_not_authorized",
Message: "simulation mode is available only through an active acceptance run",
}
}
return requested, nil
}
func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) {
var trafficErr *taskTrafficError
if !errors.As(err, &trafficErr) {
trafficErr = &taskTrafficError{
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
Message: "task traffic gate is unavailable", Err: err,
}
}
if protocol != "" {
writeProtocolError(w, protocol, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
return
}
writeErrorWithDetails(w, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
}
// getGatewayTrafficMode godoc
// @Summary 获取 Gateway 流量模式
// @Tags acceptance
// @Produce json
// @Security BearerAuth
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/traffic-mode [get]
func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
mode, err := s.store.GetGatewayTrafficMode(r.Context())
if err != nil {
writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed")
return
}
writeJSON(w, http.StatusOK, mode)
}
// createAcceptanceRun godoc
// @Summary 创建生产同构验收 Run
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param body body store.CreateAcceptanceRunInput true "验收 Run"
// @Success 201 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs [post]
func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
var input store.CreateAcceptanceRunInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
run, err := s.store.CreateAcceptanceRun(r.Context(), input)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusCreated, run)
}
// getAcceptanceRun godoc
// @Summary 获取生产同构验收 Run
// @Tags acceptance
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID} [get]
func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
run, err := s.store.GetAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusNotFound, "acceptance run not found")
return
}
writeError(w, http.StatusInternalServerError, "get acceptance run failed")
return
}
writeJSON(w, http.StatusOK, run)
}
// activateAcceptanceRun godoc
// @Summary 切换到 validation 并激活 Run
// @Tags acceptance
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/runs/{runID}/activate [post]
func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
mode, err := s.store.ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
writeAcceptanceMutationError(w, err)
return
}
writeJSON(w, http.StatusOK, mode)
}
// finishAcceptanceRun godoc
// @Summary 记录验收门禁结果
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Param body body store.FinishAcceptanceRunInput true "门禁结果"
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID}/finish [post]
func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
var input store.FinishAcceptanceRunInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
input.RunID = r.PathValue("runID")
run, err := s.store.FinishAcceptanceRun(r.Context(), input)
if err != nil {
if store.IsNotFound(err) {
writeError(w, http.StatusConflict, "acceptance run is not running")
return
}
writeError(w, http.StatusInternalServerError, "finish acceptance run failed")
return
}
writeJSON(w, http.StatusOK, run)
}
// retryAcceptanceRun godoc
// @Summary 在 validation 关闭正式流量的状态下重试失败 Run
// @Tags acceptance
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Success 200 {object} store.AcceptanceRun
// @Router /api/admin/system/acceptance/runs/{runID}/retry [post]
func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) {
run, err := s.store.RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
if err != nil {
writeAcceptanceMutationError(w, err)
return
}
writeJSON(w, http.StatusOK, run)
}
// promoteAcceptanceRun godoc
// @Summary 通过 CAS 切回 live
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/runs/{runID}/promote [post]
func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) {
var input store.PromoteAcceptanceRunInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
input.RunID = r.PathValue("runID")
mode, err := s.store.PromoteAcceptanceRun(r.Context(), input)
if err != nil {
writeAcceptanceMutationError(w, err)
return
}
writeJSON(w, http.StatusOK, mode)
}
// abortAcceptanceRun godoc
// @Summary 人工 CAS 中止验收并切回 live
// @Tags acceptance
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param runID path string true "验收 Run ID"
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
// @Success 200 {object} store.GatewayTrafficMode
// @Router /api/admin/system/acceptance/runs/{runID}/abort [post]
func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) {
var input store.PromoteAcceptanceRunInput
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
writeError(w, http.StatusBadRequest, "invalid json body")
return
}
input.RunID = r.PathValue("runID")
mode, err := s.store.AbortAcceptanceRun(r.Context(), input)
if err != nil {
writeAcceptanceMutationError(w, err)
return
}
writeJSON(w, http.StatusOK, mode)
}
func writeAcceptanceMutationError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, store.ErrAcceptanceStateConflict):
writeError(w, http.StatusConflict, err.Error())
case errors.Is(err, store.ErrAcceptancePromotionGates):
writeError(w, http.StatusPreconditionFailed, err.Error())
case store.IsNotFound(err):
writeError(w, http.StatusNotFound, "acceptance run not found")
default:
writeError(w, http.StatusInternalServerError, "acceptance state update failed")
}
}
@@ -0,0 +1,27 @@
package httpapi
import (
"errors"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
)
func TestAdmittedTaskRunModeReservesAcceptanceAndProductionSimulation(t *testing.T) {
server := &Server{cfg: config.Config{AppEnv: "production"}}
for _, runMode := range []string{"simulation", "SIMULATION", "acceptance", "acceptance_canary"} {
_, err := server.admittedTaskRunMode(taskTrafficAdmission{}, map[string]any{"runMode": runMode})
var trafficErr *taskTrafficError
if !errors.As(err, &trafficErr) || trafficErr.Status != 403 {
t.Fatalf("run mode %q error=%v", runMode, err)
}
}
for _, runMode := range []string{"acceptance", "acceptance_canary"} {
got, err := server.admittedTaskRunMode(taskTrafficAdmission{RunMode: runMode}, map[string]any{
"runMode": "production",
})
if err != nil || got != runMode {
t.Fatalf("admitted mode=%q got=%q err=%v", runMode, got, err)
}
}
}
@@ -44,14 +44,23 @@ func (s *Server) prepareAndCreateGatewayTask(
body map[string]any,
async bool,
) (store.GatewayTask, error) {
admission, err := s.authorizeTaskTraffic(r, user)
if err != nil {
return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err}
}
prepared, err := s.prepareTaskRequest(ctx, r, user, body)
if err != nil {
return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err}
}
runMode, err := s.admittedTaskRunMode(admission, prepared.Body)
if err != nil {
return store.GatewayTask{}, &gatewayTaskCreationError{Stage: gatewayTaskCreationPrepare, Err: err}
}
task, err := s.store.CreateTask(ctx, store.CreateTaskInput{
Kind: kind,
Model: model,
RunMode: runModeFromRequest(prepared.Body),
RunMode: runMode,
AcceptanceRunID: admission.AcceptanceRunID,
Async: async,
Request: prepared.Body,
ConversationID: prepared.ConversationID,
@@ -22,6 +22,7 @@ import (
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/acceptanceworkload"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/jackc/pgx/v5/pgxpool"
@@ -37,13 +38,14 @@ func TestGeminiBase64ImageEditThousandSynchronousRequests(t *testing.T) {
}
const (
taskCount = 1000
upstreamConcurrent = 64
inputVariants = 16
mediaConcurrent = 8
)
imageBytes := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_IMAGE_BYTES", 256<<10)
upstreamDelay := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_UPSTREAM_DELAY_MS", 4000)) * time.Millisecond
baselineProfile := acceptanceworkload.GeminiBaseline
taskCount := baselineProfile.Requests
imageBytes := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_IMAGE_BYTES", baselineProfile.InputBytes)
upstreamDelay := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_UPSTREAM_DELAY_MS", int(baselineProfile.DelayMin/time.Millisecond))) * time.Millisecond
maxHeapGrowth := uint64(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_MAX_HEAP_BYTES", 1<<30))
clientConnections := stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_CLIENT_CONNECTIONS", 256)
testTimeout := time.Duration(stressEnvInt(t, "AI_GATEWAY_GEMINI_STRESS_TIMEOUT_SECONDS", 1200)) * time.Second
@@ -387,7 +389,7 @@ WHERE task.model = $1
if tasks != taskCount || succeeded != taskCount || attempts != taskCount || duplicateAttemptTasks != 0 {
t.Fatalf("task results tasks=%d succeeded=%d attempts=%d duplicate_attempt_tasks=%d", tasks, succeeded, attempts, duplicateAttemptTasks)
}
if upstreamCalls.Load() != taskCount || invalidInputs.Load() != 0 {
if upstreamCalls.Load() != int64(taskCount) || invalidInputs.Load() != 0 {
t.Fatalf("upstream calls=%d invalid_inputs=%d, want %d/0", upstreamCalls.Load(), invalidInputs.Load(), taskCount)
}
if upstreamPeak.Load() > mediaConcurrent || upstreamPeak.Load() < mediaConcurrent/2 {
+12 -1
View File
@@ -117,6 +117,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
writeGeminiTaskError(http.StatusUnauthorized, "unauthorized", nil, "unauthorized")
return
}
admission, err := s.authorizeTaskTraffic(r, user)
if err != nil {
writeTaskTrafficError(w, err, clients.ProtocolGeminiGenerateContent)
return
}
releaseRequestBody, err := s.acquireMediaRequestBodySlot(r.Context())
if err != nil {
return
@@ -166,10 +171,16 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
mapping.Body = nil
releaseRequestBody()
releaseRequestBody = nil
runMode, err := s.admittedTaskRunMode(admission, prepared.Body)
if err != nil {
writeTaskTrafficError(w, err, clients.ProtocolGeminiGenerateContent)
return
}
createInput := store.CreateTaskInput{
Kind: mapping.Kind,
Model: mapping.Model,
RunMode: runModeFromRequest(prepared.Body),
RunMode: runMode,
AcceptanceRunID: admission.AcceptanceRunID,
Async: false,
Request: prepared.Body,
ConversationID: prepared.ConversationID,
+12 -1
View File
@@ -1095,6 +1095,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
writeTaskError(http.StatusUnauthorized, "unauthorized", nil)
return
}
admission, err := s.authorizeTaskTraffic(r, user)
if err != nil {
writeTaskTrafficError(w, err, targetProtocol)
return
}
body, err := s.decodeTaskRequestBody(r.Context(), w, r, kind)
if err != nil {
@@ -1147,10 +1152,16 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
return
}
runMode, err := s.admittedTaskRunMode(admission, prepared.Body)
if err != nil {
writeTaskTrafficError(w, err, targetProtocol)
return
}
createInput := store.CreateTaskInput{
Kind: kind,
Model: model,
RunMode: runModeFromRequest(prepared.Body),
RunMode: runMode,
AcceptanceRunID: admission.AcceptanceRunID,
Async: responsePlan.asyncMode,
Request: prepared.Body,
ConversationID: prepared.ConversationID,
+25 -6
View File
@@ -112,6 +112,15 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
writeKlingCompatError(w, http.StatusForbidden, "api key scope does not allow video generation", "permission_denied")
return
}
admission, err := s.authorizeTaskTraffic(r, user)
if err != nil {
targetProtocol := clients.ProtocolKlingV1Omni
if version == "v2" {
targetProtocol = clients.ProtocolKlingV2Omni
}
writeTaskTrafficError(w, err, targetProtocol)
return
}
body, externalTaskID, err := klingCompatTaskBody(version, model, native)
if err != nil {
writeKlingCompatError(w, http.StatusBadRequest, err.Error(), clients.ErrorCode(err))
@@ -122,13 +131,23 @@ func (s *Server) createKlingCompatTask(w http.ResponseWriter, r *http.Request, v
writeKlingCompatError(w, http.StatusBadRequest, err.Error(), clients.ErrorCode(err))
return
}
runMode, err := s.admittedTaskRunMode(admission, prepared.Body)
if err != nil {
targetProtocol := clients.ProtocolKlingV1Omni
if version == "v2" {
targetProtocol = clients.ProtocolKlingV2Omni
}
writeTaskTrafficError(w, err, targetProtocol)
return
}
createInput := store.CreateTaskInput{
Kind: "videos.generations",
Model: model,
ExternalTaskID: externalTaskID,
RunMode: runModeFromRequest(prepared.Body),
Async: true,
Request: prepared.Body,
Kind: "videos.generations",
Model: model,
ExternalTaskID: externalTaskID,
RunMode: runMode,
AcceptanceRunID: admission.AcceptanceRunID,
Async: true,
Request: prepared.Body,
}
if idempotencyKey, hasKey, keyErr := optionalTaskIdempotencyKey(r); keyErr != nil {
writeKlingCompatError(w, http.StatusBadRequest, "invalid Idempotency-Key", "invalid_idempotency_key")
+8
View File
@@ -217,6 +217,14 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("PATCH /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateFileStorageSettings)))
mux.Handle("GET /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getClientCustomizationSettings)))
mux.Handle("PATCH /api/admin/system/client-customization/settings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateClientCustomizationSettings)))
mux.Handle("GET /api/admin/system/acceptance/traffic-mode", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getGatewayTrafficMode)))
mux.Handle("POST /api/admin/system/acceptance/runs", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.createAcceptanceRun)))
mux.Handle("GET /api/admin/system/acceptance/runs/{runID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/activate", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.activateAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/finish", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.finishAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/promote", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.promoteAcceptanceRun)))
mux.Handle("POST /api/admin/system/acceptance/runs/{runID}/abort", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.abortAcceptanceRun)))
mux.Handle("GET /api/admin/system/identity/configuration", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityConfiguration)))
mux.Handle("POST /api/admin/system/identity/pairings", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.startIdentityPairing)))
mux.Handle("GET /api/admin/system/identity/pairings/{pairingID}", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getIdentityPairing)))
@@ -416,6 +416,11 @@ func writeVolcesCompatibleTaskError(w http.ResponseWriter, err error) {
if errors.As(err, &staged) {
err = staged.Err
}
var trafficErr *taskTrafficError
if errors.As(err, &trafficErr) {
writeVolcesError(w, trafficErr.Status, trafficErr.Message, trafficErr.Code)
return
}
var clientErr *clients.ClientError
if errors.As(err, &clientErr) && clientErr.StatusCode > 0 {
status = clientErr.StatusCode
@@ -1,6 +1,9 @@
package httpapi
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
@@ -35,3 +38,25 @@ func TestVolcesCompatibleTaskBuildsProtocolResponseFromCanonicalResult(t *testin
}
}
}
func TestVolcesCompatibilityPreservesValidationGateStatus(t *testing.T) {
recorder := httptest.NewRecorder()
writeVolcesCompatibleTaskError(recorder, &gatewayTaskCreationError{
Stage: gatewayTaskCreationPrepare,
Err: &taskTrafficError{
Status: http.StatusServiceUnavailable,
Code: "validation_in_progress",
Message: "new production tasks are paused while validation is running",
},
})
if recorder.Code != http.StatusServiceUnavailable {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
var payload VolcesErrorEnvelope
if err := json.NewDecoder(recorder.Body).Decode(&payload); err != nil {
t.Fatalf("decode error: %v", err)
}
if payload.Error.Code != "validation_in_progress" {
t.Fatalf("error=%+v", payload.Error)
}
}