feat(acceptance): 建立同构验收与弹性容量体系
实现本地三节点 K3s 同构环境、脱敏生产快照、Gemini 图片和多参考图视频协议模拟、统一验收报告及故障注入。\n\n新增 Worker 容量控制器、资源与连接预算、任务恢复保护，并将生产验收拆分为 validation 执行和人工 CAS 放量。\n\n验证包括 Go 全量测试、PostgreSQL HTTP 集成测试、go vet、OpenAPI、ShellCheck、前端检查、迁移及发布脚本测试。
This commit is contained in:
@@ -91,7 +91,10 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
pod_name = EXCLUDED.pod_name,
|
||||
site = EXCLUDED.site,
|
||||
revision = EXCLUDED.revision,
|
||||
status = 'active',
|
||||
status = CASE
|
||||
WHEN gateway_worker_instances.status = 'draining' THEN 'draining'
|
||||
ELSE 'active'
|
||||
END,
|
||||
desired_capacity = EXCLUDED.desired_capacity,
|
||||
capacity_limit = EXCLUDED.capacity_limit,
|
||||
heartbeat_at = now(),
|
||||
@@ -108,10 +111,11 @@ SET pod_uid = EXCLUDED.pod_uid,
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
SET allocated_capacity = 0,
|
||||
SET status = 'stale',
|
||||
allocated_capacity = 0,
|
||||
updated_at = now()
|
||||
WHERE status <> 'active'
|
||||
OR heartbeat_at <= now() - $1::interval`, staleAfter.String()); err != nil {
|
||||
WHERE status = 'active'
|
||||
AND heartbeat_at <= now() - $1::interval`, staleAfter.String()); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
|
||||
@@ -152,7 +156,11 @@ UPDATE gateway_worker_instances
|
||||
SET desired_capacity = $2,
|
||||
allocated_capacity = $3,
|
||||
updated_at = now()
|
||||
WHERE instance_id = $1`, worker.InstanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
WHERE instance_id = $1
|
||||
AND (
|
||||
desired_capacity IS DISTINCT FROM $2
|
||||
OR allocated_capacity IS DISTINCT FROM $3
|
||||
)`, worker.InstanceID, input.DesiredCapacity, capacity); err != nil {
|
||||
return WorkerAllocation{}, err
|
||||
}
|
||||
if worker.InstanceID == input.InstanceID {
|
||||
@@ -214,6 +222,7 @@ func (s *Store) MarkWorkerDraining(ctx context.Context, instanceID string) error
|
||||
UPDATE gateway_worker_instances
|
||||
SET status = 'draining',
|
||||
allocated_capacity = 0,
|
||||
draining_at = COALESCE(draining_at, now()),
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()
|
||||
WHERE instance_id = $1`, strings.TrimSpace(instanceID))
|
||||
@@ -226,6 +235,139 @@ WHERE instance_id = $1`, strings.TrimSpace(instanceID))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ReactivateWorkerInstance(ctx context.Context, instanceID string) error {
|
||||
result, err := s.pool.Exec(ctx, `
|
||||
UPDATE gateway_worker_instances
|
||||
SET status = 'active',
|
||||
draining_at = NULL,
|
||||
heartbeat_at = now(),
|
||||
updated_at = now()
|
||||
WHERE instance_id = $1
|
||||
AND status = 'draining'`, strings.TrimSpace(instanceID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.RowsAffected() == 0 {
|
||||
return pgx.ErrNoRows
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type WorkerInstanceRuntime struct {
|
||||
InstanceID string
|
||||
PodUID string
|
||||
PodName string
|
||||
Site string
|
||||
Revision string
|
||||
Status string
|
||||
Allocated int
|
||||
CapacityLimit int
|
||||
RunningTasks int
|
||||
ActiveLeases int
|
||||
HeartbeatAt time.Time
|
||||
DrainingAt *time.Time
|
||||
}
|
||||
|
||||
type WorkerQueueRuntime struct {
|
||||
Queued int
|
||||
Running int
|
||||
OldestWaitSeconds float64
|
||||
}
|
||||
|
||||
type CapacityDatabaseHealth struct {
|
||||
Connections int
|
||||
MaxConnections int
|
||||
SynchronousPeers int
|
||||
}
|
||||
|
||||
func (s *Store) WorkerQueueRuntime(ctx context.Context) (WorkerQueueRuntime, error) {
|
||||
var snapshot WorkerQueueRuntime
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT count(*) FILTER (WHERE status = 'queued')::int,
|
||||
count(*) FILTER (WHERE status = 'running')::int,
|
||||
COALESCE(EXTRACT(EPOCH FROM now() - MIN(created_at) FILTER (WHERE status = 'queued')), 0)::float8
|
||||
FROM gateway_tasks
|
||||
WHERE status IN ('queued', 'running')
|
||||
AND run_mode IN ('production', 'acceptance', 'acceptance_canary')`).Scan(
|
||||
&snapshot.Queued,
|
||||
&snapshot.Running,
|
||||
&snapshot.OldestWaitSeconds,
|
||||
)
|
||||
return snapshot, err
|
||||
}
|
||||
|
||||
func (s *Store) ListWorkerInstanceRuntime(ctx context.Context) ([]WorkerInstanceRuntime, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT worker.instance_id,
|
||||
worker.pod_uid,
|
||||
worker.pod_name,
|
||||
worker.site,
|
||||
worker.revision,
|
||||
worker.status,
|
||||
worker.allocated_capacity,
|
||||
worker.capacity_limit,
|
||||
count(DISTINCT task.id) FILTER (WHERE task.status = 'running')::int,
|
||||
count(DISTINCT lease.id) FILTER (WHERE lease.released_at IS NULL)::int,
|
||||
worker.heartbeat_at,
|
||||
worker.draining_at
|
||||
FROM gateway_worker_instances worker
|
||||
LEFT JOIN gateway_tasks task
|
||||
ON task.locked_by = worker.instance_id
|
||||
AND task.status IN ('queued', 'running')
|
||||
LEFT JOIN gateway_concurrency_leases lease
|
||||
ON lease.task_id = task.id
|
||||
AND lease.released_at IS NULL
|
||||
WHERE worker.status IN ('active', 'draining')
|
||||
AND worker.heartbeat_at > now() - $1::interval
|
||||
GROUP BY worker.instance_id
|
||||
ORDER BY worker.site ASC, worker.status DESC, worker.instance_id ASC`,
|
||||
runtimeWorkerStaleAfter.String(),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
instances := make([]WorkerInstanceRuntime, 0)
|
||||
for rows.Next() {
|
||||
var instance WorkerInstanceRuntime
|
||||
if err := rows.Scan(
|
||||
&instance.InstanceID,
|
||||
&instance.PodUID,
|
||||
&instance.PodName,
|
||||
&instance.Site,
|
||||
&instance.Revision,
|
||||
&instance.Status,
|
||||
&instance.Allocated,
|
||||
&instance.CapacityLimit,
|
||||
&instance.RunningTasks,
|
||||
&instance.ActiveLeases,
|
||||
&instance.HeartbeatAt,
|
||||
&instance.DrainingAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
instances = append(instances, instance)
|
||||
}
|
||||
return instances, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) CapacityDatabaseHealth(ctx context.Context) (CapacityDatabaseHealth, error) {
|
||||
var health CapacityDatabaseHealth
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT
|
||||
(SELECT count(*)::int FROM pg_stat_activity),
|
||||
current_setting('max_connections')::int,
|
||||
(SELECT count(*)::int
|
||||
FROM pg_stat_replication
|
||||
WHERE state = 'streaming'
|
||||
AND sync_state IN ('sync', 'quorum'))`).Scan(
|
||||
&health.Connections,
|
||||
&health.MaxConnections,
|
||||
&health.SynchronousPeers,
|
||||
)
|
||||
return health, err
|
||||
}
|
||||
|
||||
// YieldStaleAsyncTaskAdmissions removes waiting FIFO entries whose River job
|
||||
// is still marked running even though the task no longer owns a live execution
|
||||
// lease. The job and task remain untouched: a surviving worker can recreate
|
||||
@@ -247,9 +389,6 @@ func (s *Store) YieldStaleAsyncTaskAdmissions(
|
||||
return 0, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-stale-admission-yield', 0))`); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var yielded int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH stale AS MATERIALIZED (
|
||||
@@ -268,6 +407,7 @@ WITH stale AS MATERIALIZED (
|
||||
AND job.attempted_at <= now() - $1::interval
|
||||
ORDER BY admission.priority ASC, admission.enqueued_at ASC, admission.task_id ASC
|
||||
LIMIT $2
|
||||
FOR UPDATE OF admission SKIP LOCKED
|
||||
),
|
||||
locked AS MATERIALIZED (
|
||||
SELECT task_id,
|
||||
@@ -315,9 +455,6 @@ func (s *Store) RecoverOrphanedAsyncRiverJobs(
|
||||
return 0, err
|
||||
}
|
||||
defer rollbackTransaction(tx)
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('gateway-river-orphan-recovery', 0))`); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var recovered int64
|
||||
if err := tx.QueryRow(ctx, `
|
||||
WITH orphaned AS MATERIALIZED (
|
||||
|
||||
Reference in New Issue
Block a user