feat(acceptance): 增加生产同构媒体压力验收模式
引入动态流量门禁、隔离验收身份与协议级 Gemini/Volces 模拟器，覆盖双站点 API、Worker、PostgreSQL、River、账务、回调和媒体物化链路。 新增 P24/P28/P32 容量阶梯、Worker 强杀恢复、真实小流量 canary、CAS 放量和失败保持 validation 的生产编排；Worker 执行槽、连接池、媒体并发和双站点副本数改为环境配置。 验证：Go 全量测试、真实 PostgreSQL 迁移集成测试、迁移安全检查、OpenAPI 生成、ShellCheck、Kustomize、gofmt 和 git diff --check。
This commit is contained in:
@@ -0,0 +1,300 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
||||
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
|
||||
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
|
||||
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
|
||||
)
|
||||
|
||||
type taskTrafficAdmission struct {
|
||||
RunMode string
|
||||
AcceptanceRunID string
|
||||
}
|
||||
|
||||
type taskTrafficError struct {
|
||||
Status int
|
||||
Code string
|
||||
Message string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *taskTrafficError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
func (e *taskTrafficError) ErrorCode() string {
|
||||
return e.Code
|
||||
}
|
||||
|
||||
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
|
||||
runID, err := s.store.AuthorizeAcceptanceTask(
|
||||
r.Context(),
|
||||
r.Header.Get(acceptanceRunHeader),
|
||||
r.Header.Get(acceptanceTokenHeader),
|
||||
user,
|
||||
)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrProductionTrafficPaused):
|
||||
return taskTrafficAdmission{}, &taskTrafficError{
|
||||
Status: http.StatusServiceUnavailable, Code: "validation_in_progress",
|
||||
Message: "new production tasks are paused while validation is running", Err: err,
|
||||
}
|
||||
case errors.Is(err, store.ErrAcceptanceNotAuthorized):
|
||||
return taskTrafficAdmission{}, &taskTrafficError{
|
||||
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
|
||||
Message: "acceptance credentials do not match the active run", Err: err,
|
||||
}
|
||||
case errors.Is(err, store.ErrAcceptanceRunNotActive):
|
||||
return taskTrafficAdmission{}, &taskTrafficError{
|
||||
Status: http.StatusConflict, Code: "acceptance_run_not_active",
|
||||
Message: "acceptance headers are not valid while live traffic is enabled", Err: err,
|
||||
}
|
||||
default:
|
||||
return taskTrafficAdmission{}, &taskTrafficError{
|
||||
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
|
||||
Message: "task traffic gate is unavailable", Err: err,
|
||||
}
|
||||
}
|
||||
}
|
||||
if runID != "" {
|
||||
runMode := "acceptance"
|
||||
if strings.EqualFold(strings.TrimSpace(r.Header.Get(acceptanceUpstreamHeader)), "real") {
|
||||
runMode = "acceptance_canary"
|
||||
}
|
||||
return taskTrafficAdmission{RunMode: runMode, AcceptanceRunID: runID}, nil
|
||||
}
|
||||
return taskTrafficAdmission{RunMode: "production"}, nil
|
||||
}
|
||||
|
||||
func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) {
|
||||
if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" {
|
||||
return admission.RunMode, nil
|
||||
}
|
||||
requested := runModeFromRequest(body)
|
||||
requestedMode := strings.ToLower(strings.TrimSpace(requested))
|
||||
if requestedMode == "acceptance" || requestedMode == "acceptance_canary" {
|
||||
return "", &taskTrafficError{
|
||||
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
|
||||
Message: "acceptance mode is available only through an active acceptance run",
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(s.cfg.AppEnv), "production") && requestedMode == "simulation" {
|
||||
return "", &taskTrafficError{
|
||||
Status: http.StatusForbidden, Code: "simulation_not_authorized",
|
||||
Message: "simulation mode is available only through an active acceptance run",
|
||||
}
|
||||
}
|
||||
return requested, nil
|
||||
}
|
||||
|
||||
func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) {
|
||||
var trafficErr *taskTrafficError
|
||||
if !errors.As(err, &trafficErr) {
|
||||
trafficErr = &taskTrafficError{
|
||||
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
|
||||
Message: "task traffic gate is unavailable", Err: err,
|
||||
}
|
||||
}
|
||||
if protocol != "" {
|
||||
writeProtocolError(w, protocol, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
|
||||
return
|
||||
}
|
||||
writeErrorWithDetails(w, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
|
||||
}
|
||||
|
||||
// getGatewayTrafficMode godoc
|
||||
// @Summary 获取 Gateway 流量模式
|
||||
// @Tags acceptance
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/traffic-mode [get]
|
||||
func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
|
||||
mode, err := s.store.GetGatewayTrafficMode(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// createAcceptanceRun godoc
|
||||
// @Summary 创建生产同构验收 Run
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param body body store.CreateAcceptanceRunInput true "验收 Run"
|
||||
// @Success 201 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs [post]
|
||||
func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.CreateAcceptanceRunInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
run, err := s.store.CreateAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, run)
|
||||
}
|
||||
|
||||
// getAcceptanceRun godoc
|
||||
// @Summary 获取生产同构验收 Run
|
||||
// @Tags acceptance
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID} [get]
|
||||
func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
run, err := s.store.GetAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
if store.IsNotFound(err) {
|
||||
writeError(w, http.StatusNotFound, "acceptance run not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "get acceptance run failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, run)
|
||||
}
|
||||
|
||||
// activateAcceptanceRun godoc
|
||||
// @Summary 切换到 validation 并激活 Run
|
||||
// @Tags acceptance
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/activate [post]
|
||||
func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
mode, err := s.store.ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// finishAcceptanceRun godoc
|
||||
// @Summary 记录验收门禁结果
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Param body body store.FinishAcceptanceRunInput true "门禁结果"
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/finish [post]
|
||||
func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.FinishAcceptanceRunInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
run, err := s.store.FinishAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
if store.IsNotFound(err) {
|
||||
writeError(w, http.StatusConflict, "acceptance run is not running")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "finish acceptance run failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, run)
|
||||
}
|
||||
|
||||
// retryAcceptanceRun godoc
|
||||
// @Summary 在 validation 关闭正式流量的状态下重试失败 Run
|
||||
// @Tags acceptance
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Success 200 {object} store.AcceptanceRun
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/retry [post]
|
||||
func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
run, err := s.store.RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, run)
|
||||
}
|
||||
|
||||
// promoteAcceptanceRun godoc
|
||||
// @Summary 通过 CAS 切回 live
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/promote [post]
|
||||
func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.PromoteAcceptanceRunInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
mode, err := s.store.PromoteAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
// abortAcceptanceRun godoc
|
||||
// @Summary 人工 CAS 中止验收并切回 live
|
||||
// @Tags acceptance
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Security BearerAuth
|
||||
// @Param runID path string true "验收 Run ID"
|
||||
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
|
||||
// @Success 200 {object} store.GatewayTrafficMode
|
||||
// @Router /api/admin/system/acceptance/runs/{runID}/abort [post]
|
||||
func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
||||
var input store.PromoteAcceptanceRunInput
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid json body")
|
||||
return
|
||||
}
|
||||
input.RunID = r.PathValue("runID")
|
||||
mode, err := s.store.AbortAcceptanceRun(r.Context(), input)
|
||||
if err != nil {
|
||||
writeAcceptanceMutationError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, mode)
|
||||
}
|
||||
|
||||
func writeAcceptanceMutationError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrAcceptanceStateConflict):
|
||||
writeError(w, http.StatusConflict, err.Error())
|
||||
case errors.Is(err, store.ErrAcceptancePromotionGates):
|
||||
writeError(w, http.StatusPreconditionFailed, err.Error())
|
||||
case store.IsNotFound(err):
|
||||
writeError(w, http.StatusNotFound, "acceptance run not found")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "acceptance state update failed")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user