fix(runner): 稳定异步任务重启恢复 #12
@@ -1654,6 +1654,17 @@ WHERE m.platform_id = $1::uuid
|
|||||||
t.Fatal("task progress callback outbox should receive events")
|
t.Fatal("task progress callback outbox should receive events")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
restartModel := "worker-restart-" + suffixText
|
||||||
|
createSimulationTextPlatformModel(
|
||||||
|
t,
|
||||||
|
server.URL,
|
||||||
|
loginResponse.AccessToken,
|
||||||
|
"openai-worker-restart-"+suffixText,
|
||||||
|
"OpenAI Worker Restart",
|
||||||
|
restartModel,
|
||||||
|
1,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
var restartAsyncTask struct {
|
var restartAsyncTask struct {
|
||||||
TaskID string `json:"taskId"`
|
TaskID string `json:"taskId"`
|
||||||
Task struct {
|
Task struct {
|
||||||
@@ -1663,7 +1674,7 @@ WHERE m.platform_id = $1::uuid
|
|||||||
} `json:"task"`
|
} `json:"task"`
|
||||||
}
|
}
|
||||||
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
|
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/responses", apiKeyResponse.Secret, map[string]any{
|
||||||
"model": defaultTextModel,
|
"model": restartModel,
|
||||||
"runMode": "simulation",
|
"runMode": "simulation",
|
||||||
"simulation": true,
|
"simulation": true,
|
||||||
"simulationDurationMs": 2000,
|
"simulationDurationMs": 2000,
|
||||||
|
|||||||
@@ -156,7 +156,7 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item))
|
result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item, time.Now()))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -196,9 +196,11 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem) *river.InsertOpts {
|
func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem, now time.Time) *river.InsertOpts {
|
||||||
opts := asyncTaskInsertOpts(store.GatewayTask{ID: item.ID})
|
opts := asyncTaskInsertOpts(store.GatewayTask{ID: item.ID})
|
||||||
opts.ScheduledAt = item.NextRunAt
|
if item.NextRunAt.After(now) {
|
||||||
|
opts.ScheduledAt = item.NextRunAt
|
||||||
|
}
|
||||||
// A replacement process must not be blocked by a River row that the dead
|
// A replacement process must not be blocked by a River row that the dead
|
||||||
// process left in running state. PostgreSQL execution leases still ensure
|
// process left in running state. PostgreSQL execution leases still ensure
|
||||||
// that only one recovery job can call the upstream provider.
|
// that only one recovery job can call the upstream provider.
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAsyncTaskRecoveryInsertOptsMakesDueTaskImmediatelyAvailable(t *testing.T) {
|
||||||
|
now := time.Date(2026, time.July, 21, 3, 30, 0, 0, time.UTC)
|
||||||
|
opts := asyncTaskRecoveryInsertOpts(store.AsyncTaskQueueItem{
|
||||||
|
ID: "due-task",
|
||||||
|
NextRunAt: now.Add(-time.Second),
|
||||||
|
}, now)
|
||||||
|
|
||||||
|
if !opts.ScheduledAt.IsZero() {
|
||||||
|
t.Fatalf("due recovery task should be immediately available, scheduled at %s", opts.ScheduledAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAsyncTaskRecoveryInsertOptsPreservesFutureDelay(t *testing.T) {
|
||||||
|
now := time.Date(2026, time.July, 21, 3, 30, 0, 0, time.UTC)
|
||||||
|
nextRunAt := now.Add(30 * time.Second)
|
||||||
|
opts := asyncTaskRecoveryInsertOpts(store.AsyncTaskQueueItem{
|
||||||
|
ID: "delayed-task",
|
||||||
|
NextRunAt: nextRunAt,
|
||||||
|
}, now)
|
||||||
|
|
||||||
|
if !opts.ScheduledAt.Equal(nextRunAt) {
|
||||||
|
t.Fatalf("future recovery task should stay delayed until %s, got %s", nextRunAt, opts.ScheduledAt)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user