wangbo
132cda35d8
线上 P24 验收暴露出高频 kubectl exec 放大 K3s API 压力、门禁查询挤占关键连接池，以及 PostgreSQL 锁超时被误判为租约所有权丢失。 本次合并验收身份查询、在租约有效期内重试瞬态续期错误、修复人工审核残留 attempt，并增加滚动后 etcd 稳定窗口、节点直连指标和双站独立报告。 验证：Go 全量测试、go vet、聚焦 race、gofmt、迁移安全检查、bash -n、ShellCheck、manual release test。
466 lines
15 KiB
Go
466 lines
15 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
|
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
|
)
|
|
|
|
const (
|
|
acceptanceRunHeader = "X-EasyAI-Acceptance-Run"
|
|
acceptanceTokenHeader = "X-EasyAI-Acceptance-Token"
|
|
acceptanceUpstreamHeader = "X-EasyAI-Acceptance-Upstream"
|
|
taskTrafficAuthorizationCacheTTL = 5 * time.Second
|
|
)
|
|
|
|
type taskTrafficAuthorizationCacheKey struct {
|
|
runID string
|
|
apiKeyID string
|
|
userID string
|
|
tokenHash [sha256.Size]byte
|
|
}
|
|
|
|
type taskTrafficAuthorizationCacheEntry struct {
|
|
runID string
|
|
expiresAt time.Time
|
|
}
|
|
|
|
type taskTrafficAuthorizationCall struct {
|
|
done chan struct{}
|
|
runID string
|
|
err error
|
|
}
|
|
|
|
type taskTrafficAdmission struct {
|
|
RunMode string
|
|
AcceptanceRunID string
|
|
}
|
|
|
|
type taskTrafficError struct {
|
|
Status int
|
|
Code string
|
|
Message string
|
|
Err error
|
|
}
|
|
|
|
func (s *Server) acceptanceStore() *store.Store {
|
|
if s.coordinationStore != nil {
|
|
return s.coordinationStore
|
|
}
|
|
return s.store
|
|
}
|
|
|
|
func (e *taskTrafficError) Error() string {
|
|
return e.Message
|
|
}
|
|
|
|
func (e *taskTrafficError) ErrorCode() string {
|
|
return e.Code
|
|
}
|
|
|
|
func (s *Server) authorizeTaskTraffic(r *http.Request, user *auth.User) (taskTrafficAdmission, error) {
|
|
runID, err := s.authorizeAcceptanceTask(r.Context(), r.Header.Get(acceptanceRunHeader), r.Header.Get(acceptanceTokenHeader), user)
|
|
if err != nil {
|
|
switch {
|
|
case errors.Is(err, store.ErrProductionTrafficPaused):
|
|
return taskTrafficAdmission{}, &taskTrafficError{
|
|
Status: http.StatusServiceUnavailable, Code: "validation_in_progress",
|
|
Message: "new production tasks are paused while validation is running", Err: err,
|
|
}
|
|
case errors.Is(err, store.ErrAcceptanceNotAuthorized):
|
|
return taskTrafficAdmission{}, &taskTrafficError{
|
|
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
|
|
Message: "acceptance credentials do not match the active run", Err: err,
|
|
}
|
|
case errors.Is(err, store.ErrAcceptanceRunNotActive):
|
|
return taskTrafficAdmission{}, &taskTrafficError{
|
|
Status: http.StatusConflict, Code: "acceptance_run_not_active",
|
|
Message: "acceptance headers are not valid while live traffic is enabled", Err: err,
|
|
}
|
|
default:
|
|
return taskTrafficAdmission{}, &taskTrafficError{
|
|
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
|
|
Message: "task traffic gate is unavailable", Err: err,
|
|
}
|
|
}
|
|
}
|
|
if runID != "" {
|
|
runMode := "acceptance"
|
|
if strings.EqualFold(strings.TrimSpace(r.Header.Get(acceptanceUpstreamHeader)), "real") {
|
|
runMode = "acceptance_canary"
|
|
}
|
|
return taskTrafficAdmission{RunMode: runMode, AcceptanceRunID: runID}, nil
|
|
}
|
|
return taskTrafficAdmission{RunMode: "production"}, nil
|
|
}
|
|
|
|
// authorizeAcceptanceTask collapses concurrent validation requests for the same
|
|
// participant into one critical-pool lookup and briefly caches only successful
|
|
// validation grants. Live requests and incomplete acceptance credentials always
|
|
// consult PostgreSQL so a validation transition remains fail-closed.
|
|
func (s *Server) authorizeAcceptanceTask(ctx context.Context, runID string, token string, user *auth.User) (string, error) {
|
|
runID = strings.TrimSpace(runID)
|
|
token = strings.TrimSpace(token)
|
|
if runID == "" || token == "" || user == nil ||
|
|
strings.TrimSpace(user.APIKeyID) == "" || strings.TrimSpace(user.ID) == "" {
|
|
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
|
|
}
|
|
key := taskTrafficAuthorizationCacheKey{
|
|
runID: runID, apiKeyID: strings.TrimSpace(user.APIKeyID), userID: strings.TrimSpace(user.ID),
|
|
tokenHash: sha256.Sum256([]byte(token)),
|
|
}
|
|
return s.cachedTaskTrafficAuthorization(ctx, key, taskTrafficAuthorizationCacheTTL, func() (string, error) {
|
|
return s.acceptanceStore().AuthorizeAcceptanceTask(ctx, runID, token, user)
|
|
})
|
|
}
|
|
|
|
func (s *Server) cachedTaskTrafficAuthorization(
|
|
ctx context.Context,
|
|
key taskTrafficAuthorizationCacheKey,
|
|
ttl time.Duration,
|
|
load func() (string, error),
|
|
) (string, error) {
|
|
now := time.Now()
|
|
s.taskTrafficCacheMu.Lock()
|
|
if entry, ok := s.taskTrafficCache[key]; ok {
|
|
if now.Before(entry.expiresAt) {
|
|
s.taskTrafficCacheMu.Unlock()
|
|
return entry.runID, nil
|
|
}
|
|
delete(s.taskTrafficCache, key)
|
|
}
|
|
if call, ok := s.taskTrafficInflight[key]; ok {
|
|
s.taskTrafficCacheMu.Unlock()
|
|
select {
|
|
case <-call.done:
|
|
return call.runID, call.err
|
|
case <-ctx.Done():
|
|
return "", ctx.Err()
|
|
}
|
|
}
|
|
if s.taskTrafficInflight == nil {
|
|
s.taskTrafficInflight = make(map[taskTrafficAuthorizationCacheKey]*taskTrafficAuthorizationCall)
|
|
}
|
|
call := &taskTrafficAuthorizationCall{done: make(chan struct{})}
|
|
s.taskTrafficInflight[key] = call
|
|
s.taskTrafficCacheMu.Unlock()
|
|
|
|
call.runID, call.err = load()
|
|
|
|
s.taskTrafficCacheMu.Lock()
|
|
delete(s.taskTrafficInflight, key)
|
|
if call.err == nil && call.runID != "" {
|
|
if s.taskTrafficCache == nil {
|
|
s.taskTrafficCache = make(map[taskTrafficAuthorizationCacheKey]taskTrafficAuthorizationCacheEntry)
|
|
}
|
|
callTTL := ttl
|
|
if callTTL <= 0 {
|
|
callTTL = taskTrafficAuthorizationCacheTTL
|
|
}
|
|
s.taskTrafficCache[key] = taskTrafficAuthorizationCacheEntry{
|
|
runID: call.runID, expiresAt: time.Now().Add(callTTL),
|
|
}
|
|
}
|
|
close(call.done)
|
|
s.taskTrafficCacheMu.Unlock()
|
|
return call.runID, call.err
|
|
}
|
|
|
|
func (s *Server) admittedTaskRunMode(admission taskTrafficAdmission, body map[string]any) (string, error) {
|
|
if admission.RunMode == "acceptance" || admission.RunMode == "acceptance_canary" {
|
|
return admission.RunMode, nil
|
|
}
|
|
requested := runModeFromRequest(body)
|
|
requestedMode := strings.ToLower(strings.TrimSpace(requested))
|
|
if requestedMode == "acceptance" || requestedMode == "acceptance_canary" {
|
|
return "", &taskTrafficError{
|
|
Status: http.StatusForbidden, Code: "acceptance_not_authorized",
|
|
Message: "acceptance mode is available only through an active acceptance run",
|
|
}
|
|
}
|
|
if strings.EqualFold(strings.TrimSpace(s.cfg.AppEnv), "production") && requestedMode == "simulation" {
|
|
return "", &taskTrafficError{
|
|
Status: http.StatusForbidden, Code: "simulation_not_authorized",
|
|
Message: "simulation mode is available only through an active acceptance run",
|
|
}
|
|
}
|
|
return requested, nil
|
|
}
|
|
|
|
func writeTaskTrafficError(w http.ResponseWriter, err error, protocol string) {
|
|
var trafficErr *taskTrafficError
|
|
if !errors.As(err, &trafficErr) {
|
|
trafficErr = &taskTrafficError{
|
|
Status: http.StatusServiceUnavailable, Code: "traffic_gate_unavailable",
|
|
Message: "task traffic gate is unavailable", Err: err,
|
|
}
|
|
}
|
|
if protocol != "" {
|
|
writeProtocolError(w, protocol, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
|
|
return
|
|
}
|
|
writeErrorWithDetails(w, trafficErr.Status, trafficErr.Message, nil, trafficErr.Code)
|
|
}
|
|
|
|
// getGatewayTrafficMode godoc
|
|
// @Summary 获取 Gateway 流量模式
|
|
// @Tags acceptance
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Success 200 {object} store.GatewayTrafficMode
|
|
// @Router /api/admin/system/acceptance/traffic-mode [get]
|
|
func (s *Server) getGatewayTrafficMode(w http.ResponseWriter, r *http.Request) {
|
|
mode, err := s.acceptanceStore().GetGatewayTrafficMode(r.Context())
|
|
if err != nil {
|
|
writeError(w, http.StatusInternalServerError, "get gateway traffic mode failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, mode)
|
|
}
|
|
|
|
// pauseGatewayTraffic godoc
|
|
// @Summary 上线后硬门禁异常时通过 CAS 暂停正式新任务
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param body body store.PauseGatewayTrafficInput true "当前线上 release 与 revision CAS"
|
|
// @Success 200 {object} store.GatewayTrafficMode
|
|
// @Router /api/admin/system/acceptance/traffic-mode/pause [post]
|
|
func (s *Server) pauseGatewayTraffic(w http.ResponseWriter, r *http.Request) {
|
|
var input store.PauseGatewayTrafficInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
mode, err := s.acceptanceStore().PauseGatewayTraffic(r.Context(), input)
|
|
if err != nil {
|
|
writeAcceptanceMutationError(w, err)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, mode)
|
|
}
|
|
|
|
// listCapacityProfiles godoc
|
|
// @Summary 获取经生产同构验收认证的容量配置
|
|
// @Tags acceptance
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Success 200 {array} store.CapacityProfile
|
|
// @Router /api/admin/system/acceptance/capacity-profiles [get]
|
|
func (s *Server) listCapacityProfiles(w http.ResponseWriter, r *http.Request) {
|
|
profiles, err := s.acceptanceStore().ListCapacityProfiles(r.Context())
|
|
if err != nil {
|
|
writeError(w, http.StatusInternalServerError, "list capacity profiles failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, profiles)
|
|
}
|
|
|
|
// createAcceptanceRun godoc
|
|
// @Summary 创建生产同构验收 Run
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param body body store.CreateAcceptanceRunInput true "验收 Run"
|
|
// @Success 201 {object} store.AcceptanceRun
|
|
// @Router /api/admin/system/acceptance/runs [post]
|
|
func (s *Server) createAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
var input store.CreateAcceptanceRunInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
run, err := s.acceptanceStore().CreateAcceptanceRun(r.Context(), input)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusCreated, run)
|
|
}
|
|
|
|
// getAcceptanceRun godoc
|
|
// @Summary 获取生产同构验收 Run
|
|
// @Tags acceptance
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Success 200 {object} store.AcceptanceRun
|
|
// @Router /api/admin/system/acceptance/runs/{runID} [get]
|
|
func (s *Server) getAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
run, err := s.acceptanceStore().GetAcceptanceRun(r.Context(), r.PathValue("runID"))
|
|
if err != nil {
|
|
if store.IsNotFound(err) {
|
|
writeError(w, http.StatusNotFound, "acceptance run not found")
|
|
return
|
|
}
|
|
writeError(w, http.StatusInternalServerError, "get acceptance run failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, run)
|
|
}
|
|
|
|
// activateAcceptanceRun godoc
|
|
// @Summary 切换到 validation 并激活 Run
|
|
// @Tags acceptance
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Success 200 {object} store.GatewayTrafficMode
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/activate [post]
|
|
func (s *Server) activateAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
mode, err := s.acceptanceStore().ActivateAcceptanceRun(r.Context(), r.PathValue("runID"))
|
|
if err != nil {
|
|
writeAcceptanceMutationError(w, err)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, mode)
|
|
}
|
|
|
|
// stageAcceptanceCapacityProfiles godoc
|
|
// @Summary 为当前 validation Run 暂存 80% 容量门禁
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Param body body store.StageAcceptanceCapacityProfilesInput true "待验证容量配置"
|
|
// @Success 200 {object} store.AcceptanceRun
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/capacity-profiles [post]
|
|
func (s *Server) stageAcceptanceCapacityProfiles(w http.ResponseWriter, r *http.Request) {
|
|
var input store.StageAcceptanceCapacityProfilesInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
input.RunID = r.PathValue("runID")
|
|
run, err := s.acceptanceStore().StageAcceptanceCapacityProfiles(r.Context(), input)
|
|
if err != nil {
|
|
switch {
|
|
case store.IsNotFound(err):
|
|
writeError(w, http.StatusConflict, "acceptance run is not running")
|
|
default:
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
}
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, run)
|
|
}
|
|
|
|
// finishAcceptanceRun godoc
|
|
// @Summary 记录验收门禁结果
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Param body body store.FinishAcceptanceRunInput true "门禁结果"
|
|
// @Success 200 {object} store.AcceptanceRun
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/finish [post]
|
|
func (s *Server) finishAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
var input store.FinishAcceptanceRunInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
input.RunID = r.PathValue("runID")
|
|
run, err := s.acceptanceStore().FinishAcceptanceRun(r.Context(), input)
|
|
if err != nil {
|
|
if store.IsNotFound(err) {
|
|
writeError(w, http.StatusConflict, "acceptance run is not running")
|
|
return
|
|
}
|
|
writeError(w, http.StatusInternalServerError, "finish acceptance run failed")
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, run)
|
|
}
|
|
|
|
// retryAcceptanceRun godoc
|
|
// @Summary 在 validation 关闭正式流量的状态下重试失败 Run
|
|
// @Tags acceptance
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Success 200 {object} store.AcceptanceRun
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/retry [post]
|
|
func (s *Server) retryAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
run, err := s.acceptanceStore().RetryAcceptanceRun(r.Context(), r.PathValue("runID"))
|
|
if err != nil {
|
|
writeAcceptanceMutationError(w, err)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, run)
|
|
}
|
|
|
|
// promoteAcceptanceRun godoc
|
|
// @Summary 通过 CAS 切回 live
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
|
|
// @Success 200 {object} store.GatewayTrafficMode
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/promote [post]
|
|
func (s *Server) promoteAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
var input store.PromoteAcceptanceRunInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
input.RunID = r.PathValue("runID")
|
|
mode, err := s.acceptanceStore().PromoteAcceptanceRun(r.Context(), input)
|
|
if err != nil {
|
|
writeAcceptanceMutationError(w, err)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, mode)
|
|
}
|
|
|
|
// abortAcceptanceRun godoc
|
|
// @Summary 人工 CAS 中止验收并切回 live
|
|
// @Tags acceptance
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Security BearerAuth
|
|
// @Param runID path string true "验收 Run ID"
|
|
// @Param body body store.PromoteAcceptanceRunInput true "线上 release 与 digest CAS"
|
|
// @Success 200 {object} store.GatewayTrafficMode
|
|
// @Router /api/admin/system/acceptance/runs/{runID}/abort [post]
|
|
func (s *Server) abortAcceptanceRun(w http.ResponseWriter, r *http.Request) {
|
|
var input store.PromoteAcceptanceRunInput
|
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
|
writeError(w, http.StatusBadRequest, "invalid json body")
|
|
return
|
|
}
|
|
input.RunID = r.PathValue("runID")
|
|
mode, err := s.acceptanceStore().AbortAcceptanceRun(r.Context(), input)
|
|
if err != nil {
|
|
writeAcceptanceMutationError(w, err)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, mode)
|
|
}
|
|
|
|
func writeAcceptanceMutationError(w http.ResponseWriter, err error) {
|
|
switch {
|
|
case errors.Is(err, store.ErrAcceptanceStateConflict):
|
|
writeError(w, http.StatusConflict, err.Error())
|
|
case errors.Is(err, store.ErrAcceptancePromotionGates):
|
|
writeError(w, http.StatusPreconditionFailed, err.Error())
|
|
case store.IsNotFound(err):
|
|
writeError(w, http.StatusNotFound, "acceptance run not found")
|
|
default:
|
|
writeError(w, http.StatusInternalServerError, "acceptance state update failed")
|
|
}
|
|
}
|