fix(billing): 封住发布前计费竞态
ci / verify (pull_request) Failing after 8s
ci / verify (pull_request) Failing after 8s
阻止上游提交状态不明的任务被租约接管后重复执行,并为人工复核保留可操作的结算记录。 将生产提交绑定到当前估价签名,统一复用预处理快照,并补强规则形状、定点溢出与历史规则兼容校验。 已通过 PostgreSQL 16 集成测试、Go 全量测试与静态检查、前端测试与构建、OpenAPI、依赖审计、镜像、迁移、流水线和 SemVer 门禁。
This commit is contained in:
@@ -62,6 +62,149 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
|
||||
if err != nil || finished.ErrorCode != "new_worker" {
|
||||
t.Fatalf("new worker terminal task=%+v err=%v", finished, err)
|
||||
}
|
||||
if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, secondToken, 5*time.Minute); !errors.Is(err, ErrTaskExecutionFinished) {
|
||||
t.Fatalf("terminal task renewal error=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
|
||||
user := &auth.User{ID: "billing-review-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID}
|
||||
created, err := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
|
||||
Request: map[string]any{"model": "billing-v2-model"},
|
||||
}, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
|
||||
})
|
||||
|
||||
firstToken := uuid.NewString()
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, firstToken, 5*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.ID, AttemptNo: 1, Status: "running", RequestSnapshot: map[string]any{"model": created.Model},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); !errors.Is(err, ErrTaskExecutionManualReview) {
|
||||
t.Fatalf("ambiguous submission takeover error=%v", err)
|
||||
}
|
||||
|
||||
review, err := db.GetTask(ctx, created.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
|
||||
t.Fatalf("manual review task=%+v", review)
|
||||
}
|
||||
var outboxStatus string
|
||||
var outboxAction string
|
||||
var reviewReason string
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT status, action, COALESCE(manual_review_reason, '')
|
||||
FROM settlement_outbox
|
||||
WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&outboxStatus, &outboxAction, &reviewReason); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_submission_unknown" {
|
||||
t.Fatalf("review outbox status=%s action=%s reason=%s", outboxStatus, outboxAction, reviewReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredExecutionLeaseCanResumeAfterKnownRejectedResponse(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
_, gatewayUserID := seedWalletReservationUser(t, ctx, db)
|
||||
user := &auth.User{ID: "billing-known-response-" + uuid.NewString(), GatewayUserID: gatewayUserID}
|
||||
created, err := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
|
||||
Request: map[string]any{"model": "billing-v2-model"},
|
||||
}, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
|
||||
})
|
||||
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "response_received"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.FinishTaskAttempt(ctx, FinishTaskAttemptInput{AttemptID: attemptID, Status: "failed", ErrorCode: "upstream_rejected"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
|
||||
t.Fatalf("known rejected response should remain retryable: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
_, gatewayUserID := seedWalletReservationUser(t, ctx, db)
|
||||
user := &auth.User{ID: "billing-direct-review-" + uuid.NewString(), GatewayUserID: gatewayUserID}
|
||||
created, err := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
|
||||
Request: map[string]any{"model": "billing-v2-model"},
|
||||
}, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
|
||||
})
|
||||
token := uuid.NewString()
|
||||
if _, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.FinishTaskManualReview(ctx, FinishTaskManualReviewInput{
|
||||
TaskID: created.ID, ExecutionToken: token, AttemptID: attemptID, TaskStatus: "failed",
|
||||
Code: "upstream_submission_unknown", Message: "upstream submission result is unknown",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var visible bool
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM settlement_outbox
|
||||
WHERE task_id=$1::uuid AND status='manual_review'
|
||||
AND action='release' AND manual_review_reason='upstream_submission_unknown'
|
||||
)`, created.ID).Scan(&visible); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !visible {
|
||||
t.Fatal("manual review billing record is not visible in settlement outbox")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBillingSettlementStaleTakeoverDebitsExactlyOnce(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user