feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -405,10 +405,35 @@ func TestBillingSettlementStaleTakeoverDebitsExactlyOnce(t *testing.T) {
|
||||
if _, err := db.FinishTaskSuccess(ctx, FinishTaskSuccessInput{
|
||||
TaskID: created.ID, ExecutionToken: token, Result: map[string]any{"ok": true},
|
||||
FinalChargeAmountText: amount, BillingCurrency: "resource",
|
||||
PricingSnapshot: map[string]any{"pricingVersion": "effective-pricing-v2"},
|
||||
PricingSnapshot: map[string]any{"pricingVersion": "effective-pricing-v2"},
|
||||
CompletionCallbackURL: "https://callback.invalid/task",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var completionEvents int
|
||||
var completionCallbacks int
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*)
|
||||
FROM gateway_task_events
|
||||
WHERE task_id=$1::uuid
|
||||
AND event_type='task.completed'
|
||||
AND status='succeeded'
|
||||
AND payload='{}'::jsonb),
|
||||
(SELECT count(*)
|
||||
FROM gateway_task_callback_outbox callback
|
||||
JOIN gateway_task_events event ON event.id=callback.event_id
|
||||
WHERE callback.task_id=$1::uuid
|
||||
AND callback.callback_url='https://callback.invalid/task'
|
||||
AND callback.seq=event.seq
|
||||
AND callback.payload='{}'::jsonb)`,
|
||||
created.ID,
|
||||
).Scan(&completionEvents, &completionCallbacks); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if completionEvents != 1 || completionCallbacks != 1 {
|
||||
t.Fatalf("atomic completion history=%d callbacks=%d, want 1/1", completionEvents, completionCallbacks)
|
||||
}
|
||||
|
||||
firstClaims, err := db.ClaimBillingSettlements(ctx, "worker-one", BillingSettlementBatchSize, BillingSettlementLockTimeout)
|
||||
if err != nil {
|
||||
@@ -468,6 +493,133 @@ WHERE gateway_user_id=$1::uuid AND reference_id=$2`, gatewayUserID, created.ID).
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiredSynchronousExecutionRecoveryReleasesBillingReservation(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
|
||||
user := &auth.User{
|
||||
ID: "billing-recovery-" + uuid.NewString(),
|
||||
GatewayUserID: gatewayUserID,
|
||||
GatewayTenantID: tenantID,
|
||||
}
|
||||
if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{
|
||||
GatewayUserID: gatewayUserID,
|
||||
Currency: "resource",
|
||||
Balance: 10,
|
||||
Reason: "billing recovery test",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
created, err := db.CreateTask(ctx, CreateTaskInput{
|
||||
Kind: "images.generations",
|
||||
Model: "billing-v2-model",
|
||||
RunMode: "production",
|
||||
Request: map[string]any{"model": "billing-v2-model"},
|
||||
}, user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
cleanupCtx := context.Background()
|
||||
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid`, gatewayUserID)
|
||||
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
|
||||
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid`, gatewayUserID)
|
||||
})
|
||||
|
||||
token := uuid.NewString()
|
||||
claimed, err := db.ClaimTaskExecutionForOwner(ctx, created.ID, token, 5*time.Minute, "expired-worker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
amount := "1.000000000"
|
||||
reservations, err := db.ReserveTaskBilling(ctx, claimed, user, nil, map[string]any{
|
||||
"pricingVersion": "billing-recovery-v1",
|
||||
"reservationAmount": amount,
|
||||
"currency": "resource",
|
||||
})
|
||||
if err != nil || len(reservations) != 1 {
|
||||
t.Fatalf("reserve=%+v err=%v", reservations, err)
|
||||
}
|
||||
if _, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
|
||||
TaskID: created.ID,
|
||||
ExecutionToken: token,
|
||||
AttemptNo: 1,
|
||||
Status: "running",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := db.pool.Exec(ctx, `
|
||||
UPDATE gateway_tasks
|
||||
SET execution_lease_expires_at=now()-interval '1 second'
|
||||
WHERE id=$1::uuid`, created.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
recovery, err := db.RecoverInterruptedRuntimeState(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recovery.FailedTasks != 1 || recovery.FailedAttempts != 1 {
|
||||
t.Fatalf("recovery=%+v, want one failed task and attempt", recovery)
|
||||
}
|
||||
var taskStatus, billingStatus, outboxStatus string
|
||||
var outboxCount int
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT task.status,
|
||||
task.billing_status,
|
||||
count(outbox.id),
|
||||
COALESCE(min(outbox.status), '')
|
||||
FROM gateway_tasks task
|
||||
LEFT JOIN settlement_outbox outbox
|
||||
ON outbox.task_id=task.id
|
||||
AND outbox.event_type='task.billing.release'
|
||||
AND outbox.action='release'
|
||||
WHERE task.id=$1::uuid
|
||||
GROUP BY task.status, task.billing_status`,
|
||||
created.ID,
|
||||
).Scan(&taskStatus, &billingStatus, &outboxCount, &outboxStatus); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if taskStatus != "failed" || billingStatus != "pending" || outboxCount != 1 || outboxStatus != "pending" {
|
||||
t.Fatalf(
|
||||
"recovered task status=%s billing=%s release_outbox=%d/%s",
|
||||
taskStatus,
|
||||
billingStatus,
|
||||
outboxCount,
|
||||
outboxStatus,
|
||||
)
|
||||
}
|
||||
|
||||
claims, err := db.ClaimBillingSettlements(ctx, "billing-recovery-worker", BillingSettlementBatchSize, BillingSettlementLockTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := settlementForTask(t, claims, created.ID)
|
||||
if release.Action != "release" {
|
||||
t.Fatalf("settlement action=%s, want release", release.Action)
|
||||
}
|
||||
if err := db.ProcessBillingSettlement(ctx, release); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var releasedExactly bool
|
||||
if err := db.pool.QueryRow(ctx, `
|
||||
SELECT account.balance=10::numeric
|
||||
AND account.frozen_balance=0::numeric
|
||||
AND task.reservation_amount=0::numeric
|
||||
AND task.billing_status='released'
|
||||
FROM gateway_wallet_accounts account
|
||||
JOIN gateway_tasks task ON task.gateway_user_id=account.gateway_user_id
|
||||
WHERE task.id=$1::uuid
|
||||
AND account.currency='resource'`,
|
||||
created.ID,
|
||||
).Scan(&releasedExactly); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !releasedExactly {
|
||||
t.Fatal("expired synchronous execution did not release its wallet reservation exactly once")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseTaskBillingReservationsPreservesNineDecimalPlaces(t *testing.T) {
|
||||
db := billingV2IntegrationStore(t)
|
||||
ctx := context.Background()
|
||||
|
||||
Reference in New Issue
Block a user