fix(runner): 稳定异步任务重启恢复 #12

Merged
chengcheng merged 1 commits from codex/billing-v2-ci-stability into main 2026-07-21 11:43:00 +08:00
3 changed files with 50 additions and 4 deletions
@@ -1654,6 +1654,17 @@ WHERE m.platform_id = $1::uuid
t.Fatal("task progress callback outbox should receive events")
}
restartModel := "worker-restart-" + suffixText
createSimulationTextPlatformModel(
t,
server.URL,
loginResponse.AccessToken,
"openai-worker-restart-"+suffixText,
"OpenAI Worker Restart",
restartModel,
1,
nil,
)
var restartAsyncTask struct {
TaskID string `json:"taskId"`
Task struct {
@@ -1663,7 +1674,7 @@ WHERE m.platform_id = $1::uuid
} `json:"task"`
}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
"model": defaultTextModel,
"model": restartModel,
"runMode": "simulation",
"simulation": true,
"simulationDurationMs": 2000,
+5 -3
View File
@@ -156,7 +156,7 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
return err
}
for _, item := range items {
result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item))
result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item, time.Now()))
if err != nil {
return err
}
@@ -196,9 +196,11 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
}
}
func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem) *river.InsertOpts {
func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem, now time.Time) *river.InsertOpts {
opts := asyncTaskInsertOpts(store.GatewayTask{ID: item.ID})
opts.ScheduledAt = item.NextRunAt
if item.NextRunAt.After(now) {
opts.ScheduledAt = item.NextRunAt
}
// A replacement process must not be blocked by a River row that the dead
// process left in running state. PostgreSQL execution leases still ensure
// that only one recovery job can call the upstream provider.
@@ -0,0 +1,33 @@
package runner
import (
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestAsyncTaskRecoveryInsertOptsMakesDueTaskImmediatelyAvailable(t *testing.T) {
now := time.Date(2026, time.July, 21, 3, 30, 0, 0, time.UTC)
opts := asyncTaskRecoveryInsertOpts(store.AsyncTaskQueueItem{
ID: "due-task",
NextRunAt: now.Add(-time.Second),
}, now)
if !opts.ScheduledAt.IsZero() {
t.Fatalf("due recovery task should be immediately available, scheduled at %s", opts.ScheduledAt)
}
}
func TestAsyncTaskRecoveryInsertOptsPreservesFutureDelay(t *testing.T) {
now := time.Date(2026, time.July, 21, 3, 30, 0, 0, time.UTC)
nextRunAt := now.Add(30 * time.Second)
opts := asyncTaskRecoveryInsertOpts(store.AsyncTaskQueueItem{
ID: "delayed-task",
NextRunAt: nextRunAt,
}, now)
if !opts.ScheduledAt.Equal(nextRunAt) {
t.Fatalf("future recovery task should stay delayed until %s, got %s", nextRunAt, opts.ScheduledAt)
}
}