feat(admin): 优化后台运维与任务管理

完善平台、基准模型、定价规则和实时负载的紧凑查询与滚动展示,并固定关键操作列。

新增管理员任务记录、批量计费结算和用户组限流、额度及模型权限维护能力,同时补充任务脱敏、查询索引与 OpenAPI 契约。

验证:pnpm openapi、Go 全量测试、pnpm lint、pnpm test、pnpm build、gofmt 与差异格式检查均通过。
This commit is contained in:
2026-07-25 01:32:49 +08:00
parent b29d539d49
commit de10875439
38 changed files with 5015 additions and 425 deletions
@@ -0,0 +1,94 @@
package store
import "strings"
const maskedAdminTaskValue = "***"
func MaskAdminGatewayTask(task AdminGatewayTask) AdminGatewayTask {
task.Request = maskSensitiveObject(task.Request)
task.Result = maskSensitiveObject(task.Result)
task.Billings = maskSensitiveArray(task.Billings)
task.Usage = maskSensitiveObject(task.Usage)
task.Metrics = maskSensitiveObject(task.Metrics)
task.BillingSummary = maskSensitiveObject(task.BillingSummary)
task.PricingSnapshot = maskSensitiveObject(task.PricingSnapshot)
task.CompatibilitySubmitHeaders = maskSensitiveObject(task.CompatibilitySubmitHeaders)
task.CompatibilitySubmitBody = maskSensitiveObject(task.CompatibilitySubmitBody)
task.Attempts = append([]TaskAttempt(nil), task.Attempts...)
for index := range task.Attempts {
task.Attempts[index].Usage = maskSensitiveObject(task.Attempts[index].Usage)
task.Attempts[index].Metrics = maskSensitiveObject(task.Attempts[index].Metrics)
task.Attempts[index].RequestSnapshot = maskSensitiveObject(task.Attempts[index].RequestSnapshot)
task.Attempts[index].ResponseSnapshot = maskSensitiveObject(task.Attempts[index].ResponseSnapshot)
task.Attempts[index].PricingSnapshot = maskSensitiveObject(task.Attempts[index].PricingSnapshot)
}
return task
}
func MaskTaskParamPreprocessingLogs(items []TaskParamPreprocessingLog) []TaskParamPreprocessingLog {
masked := make([]TaskParamPreprocessingLog, len(items))
for index, item := range items {
item.ActualInput = maskSensitiveObject(item.ActualInput)
item.ConvertedOutput = maskSensitiveObject(item.ConvertedOutput)
item.Changes = maskSensitiveArray(item.Changes)
item.ModelSnapshot = maskSensitiveObject(item.ModelSnapshot)
masked[index] = item
}
return masked
}
func maskSensitiveObject(value map[string]any) map[string]any {
if value == nil {
return nil
}
masked, _ := maskSensitiveValue(value).(map[string]any)
return masked
}
func maskSensitiveArray(value []any) []any {
if value == nil {
return nil
}
masked, _ := maskSensitiveValue(value).([]any)
return masked
}
func maskSensitiveValue(value any) any {
switch typed := value.(type) {
case map[string]any:
masked := make(map[string]any, len(typed))
for key, item := range typed {
if adminTaskSensitiveKey(key) {
masked[key] = maskedAdminTaskValue
continue
}
masked[key] = maskSensitiveValue(item)
}
return masked
case []any:
masked := make([]any, len(typed))
for index, item := range typed {
masked[index] = maskSensitiveValue(item)
}
return masked
default:
return value
}
}
func adminTaskSensitiveKey(key string) bool {
normalized := strings.NewReplacer("_", "", "-", "", ".", "", " ", "").Replace(strings.ToLower(strings.TrimSpace(key)))
switch normalized {
case "authorization", "proxyauthorization", "apikey", "xapikey", "accesstoken", "refreshtoken",
"idtoken", "token", "secret", "clientsecret", "password", "passwd", "cookie", "setcookie",
"credentials", "credential", "privatekey":
return true
default:
return strings.HasSuffix(normalized, "apikey") ||
strings.HasSuffix(normalized, "accesstoken") ||
strings.HasSuffix(normalized, "refreshtoken") ||
strings.HasSuffix(normalized, "clientsecret") ||
strings.HasSuffix(normalized, "password") ||
strings.HasSuffix(normalized, "privatekey")
}
}
@@ -0,0 +1,87 @@
package store
import (
"strings"
"testing"
)
func TestMaskAdminGatewayTaskRecursivelyMasksSecretsWithoutMutatingSource(t *testing.T) {
source := AdminGatewayTask{
GatewayTask: GatewayTask{
Request: map[string]any{
"model": "example",
"headers": map[string]any{
"Authorization": "Bearer private",
"X-Api-Key": "secret-key",
},
"usage": map[string]any{"input_tokens": float64(12)},
},
Result: map[string]any{
"nested": []any{map[string]any{"password": "private-password"}},
},
Attempts: []TaskAttempt{{
RequestSnapshot: map[string]any{"client_secret": "private-client-secret"},
}},
},
}
masked := MaskAdminGatewayTask(source)
headers := masked.Request["headers"].(map[string]any)
if headers["Authorization"] != maskedAdminTaskValue || headers["X-Api-Key"] != maskedAdminTaskValue {
t.Fatalf("sensitive headers were not masked: %#v", headers)
}
if masked.Request["usage"].(map[string]any)["input_tokens"] != float64(12) {
t.Fatalf("token usage must remain visible: %#v", masked.Request)
}
nested := masked.Result["nested"].([]any)[0].(map[string]any)
if nested["password"] != maskedAdminTaskValue {
t.Fatalf("nested password was not masked: %#v", nested)
}
if masked.Attempts[0].RequestSnapshot["client_secret"] != maskedAdminTaskValue {
t.Fatalf("attempt snapshot secret was not masked: %#v", masked.Attempts[0])
}
if source.Request["headers"].(map[string]any)["Authorization"] != "Bearer private" {
t.Fatal("masking mutated the source request")
}
if source.Attempts[0].RequestSnapshot["client_secret"] != "private-client-secret" {
t.Fatal("masking mutated the source attempts")
}
}
func TestBuildAdminTaskWhereUsesSharedPlaceholdersAndAllFilters(t *testing.T) {
where, args := buildAdminTaskWhere(AdminTaskListFilter{
Query: "needle",
GatewayTenant: "00000000-0000-0000-0000-000000000001",
GatewayUser: "00000000-0000-0000-0000-000000000002",
UserGroup: "00000000-0000-0000-0000-000000000003",
Status: "failed",
Platform: "00000000-0000-0000-0000-000000000004",
Model: "model-a",
ModelType: "image_generate",
RunMode: "production",
BillingStatus: "settled",
APIKey: "ops",
})
sql := strings.Join(where, "\n")
if strings.Contains(sql, "%!") || strings.Contains(sql, "$%d") {
t.Fatalf("SQL contains an unresolved placeholder: %s", sql)
}
if len(args) != 11 {
t.Fatalf("argument count=%d, want 11", len(args))
}
if strings.Count(sql, "$1") < 10 {
t.Fatalf("keyword search should reuse one placeholder, got: %s", sql)
}
for _, fragment := range []string{
"t.gateway_tenant_id",
"t.gateway_user_id",
"t.user_group_id",
"platform_attempt.platform_id",
"t.billing_status",
"t.api_key_prefix",
} {
if !strings.Contains(sql, fragment) {
t.Fatalf("SQL is missing %q: %s", fragment, sql)
}
}
}
+385
View File
@@ -0,0 +1,385 @@
package store
import (
"context"
"fmt"
"strings"
"time"
)
type AdminTaskListFilter struct {
Query string
GatewayTenant string
GatewayUser string
UserGroup string
Status string
Platform string
Model string
ModelType string
RunMode string
BillingStatus string
APIKey string
CreatedFrom *time.Time
CreatedTo *time.Time
Page int
PageSize int
}
type AdminTaskUserSummary struct {
ID string `json:"id"`
Username string `json:"username,omitempty"`
DisplayName string `json:"displayName,omitempty"`
Email string `json:"email,omitempty"`
Source string `json:"source,omitempty"`
}
type AdminTaskTenantSummary struct {
ID string `json:"id"`
TenantKey string `json:"tenantKey,omitempty"`
Name string `json:"name,omitempty"`
}
type AdminTaskPlatformSummary struct {
ID string `json:"id"`
Name string `json:"name,omitempty"`
Provider string `json:"provider,omitempty"`
}
type AdminTaskContext struct {
User *AdminTaskUserSummary `json:"user,omitempty"`
Tenant *AdminTaskTenantSummary `json:"tenant,omitempty"`
LatestPlatform *AdminTaskPlatformSummary `json:"latestPlatform,omitempty"`
}
type AdminGatewayTask struct {
GatewayTask
AdminContext AdminTaskContext `json:"adminContext"`
}
type AdminTaskListResult struct {
Items []AdminGatewayTask
Total int
Page int
PageSize int
}
func (s *Store) ListAdminTasks(ctx context.Context, filter AdminTaskListFilter) (AdminTaskListResult, error) {
page := filter.Page
if page <= 0 {
page = 1
}
pageSize := filter.PageSize
if pageSize <= 0 {
pageSize = 50
}
if pageSize > 100 {
pageSize = 100
}
where, args := buildAdminTaskWhere(filter)
var total int
if err := s.pool.QueryRow(ctx, `
SELECT count(*)
FROM gateway_tasks t
WHERE `+strings.Join(where, "\n AND "), args...).Scan(&total); err != nil {
return AdminTaskListResult{}, err
}
queryArgs := append([]any{}, args...)
queryArgs = append(queryArgs, pageSize, (page-1)*pageSize)
rows, err := s.pool.Query(ctx, `
SELECT t.id::text
FROM gateway_tasks t
WHERE `+strings.Join(where, "\n AND ")+`
ORDER BY t.created_at DESC, t.id DESC
LIMIT $`+fmt.Sprint(len(args)+1)+` OFFSET $`+fmt.Sprint(len(args)+2), queryArgs...)
if err != nil {
return AdminTaskListResult{}, err
}
defer rows.Close()
taskIDs := make([]string, 0, pageSize)
for rows.Next() {
var taskID string
if err := rows.Scan(&taskID); err != nil {
return AdminTaskListResult{}, err
}
taskIDs = append(taskIDs, taskID)
}
if err := rows.Err(); err != nil {
return AdminTaskListResult{}, err
}
items, err := s.loadAdminTasksByIDs(ctx, taskIDs)
if err != nil {
return AdminTaskListResult{}, err
}
return AdminTaskListResult{
Items: items,
Total: total,
Page: page,
PageSize: pageSize,
}, nil
}
func (s *Store) GetAdminTask(ctx context.Context, taskID string) (AdminGatewayTask, error) {
task, err := s.GetTask(ctx, taskID)
if err != nil {
return AdminGatewayTask{}, err
}
contexts, err := s.loadAdminTaskContexts(ctx, []string{task.ID})
if err != nil {
return AdminGatewayTask{}, err
}
return adminTaskWithContext(task, contexts[task.ID]), nil
}
func buildAdminTaskWhere(filter AdminTaskListFilter) ([]string, []any) {
where := []string{"TRUE"}
args := make([]any, 0, 14)
add := func(clause string, value any) {
args = append(args, value)
placeholder := fmt.Sprintf("$%d", len(args))
where = append(where, strings.ReplaceAll(clause, "$%d", placeholder))
}
if query := strings.TrimSpace(filter.Query); query != "" {
add(`(
t.id::text ILIKE $%d
OR COALESCE(t.request_id, '') ILIKE $%d
OR t.kind ILIKE $%d
OR t.model ILIKE $%d
OR COALESCE(t.requested_model, '') ILIKE $%d
OR COALESCE(t.resolved_model, '') ILIKE $%d
OR COALESCE(t.api_key_id, '') ILIKE $%d
OR COALESCE(t.api_key_name, '') ILIKE $%d
OR COALESCE(t.api_key_prefix, '') ILIKE $%d
OR COALESCE(t.model_type, '') ILIKE $%d
OR EXISTS (
SELECT 1
FROM gateway_users admin_user
WHERE admin_user.id = t.gateway_user_id
AND (
admin_user.username ILIKE $%d
OR COALESCE(admin_user.display_name, '') ILIKE $%d
OR COALESCE(admin_user.email, '') ILIKE $%d
OR admin_user.user_key ILIKE $%d
)
)
OR EXISTS (
SELECT 1
FROM gateway_tenants admin_tenant
WHERE admin_tenant.id = t.gateway_tenant_id
AND (
admin_tenant.name ILIKE $%d
OR admin_tenant.tenant_key ILIKE $%d
)
)
OR EXISTS (
SELECT 1
FROM gateway_task_attempts admin_attempt
LEFT JOIN integration_platforms admin_platform ON admin_platform.id = admin_attempt.platform_id
LEFT JOIN platform_models admin_model ON admin_model.id = admin_attempt.platform_model_id
WHERE admin_attempt.task_id = t.id
AND (
COALESCE(admin_platform.name, '') ILIKE $%d
OR COALESCE(admin_platform.internal_name, '') ILIKE $%d
OR COALESCE(admin_platform.provider, '') ILIKE $%d
OR COALESCE(admin_model.model_name, '') ILIKE $%d
OR COALESCE(admin_model.provider_model_name, '') ILIKE $%d
OR COALESCE(admin_model.model_alias, '') ILIKE $%d
)
)
)`, "%"+query+"%")
}
if value := strings.TrimSpace(filter.GatewayTenant); value != "" {
add("t.gateway_tenant_id = $%d::uuid", value)
}
if value := strings.TrimSpace(filter.GatewayUser); value != "" {
add("t.gateway_user_id = $%d::uuid", value)
}
if value := strings.TrimSpace(filter.UserGroup); value != "" {
add("t.user_group_id = $%d::uuid", value)
}
if value := strings.TrimSpace(filter.Status); value != "" {
add("t.status = $%d", value)
}
if value := strings.TrimSpace(filter.Platform); value != "" {
add(`EXISTS (
SELECT 1
FROM gateway_task_attempts platform_attempt
WHERE platform_attempt.task_id = t.id
AND platform_attempt.platform_id = $%d::uuid
)`, value)
}
if value := strings.TrimSpace(filter.Model); value != "" {
add(`(
t.model = $%d
OR COALESCE(t.requested_model, '') = $%d
OR COALESCE(t.resolved_model, '') = $%d
OR EXISTS (
SELECT 1
FROM gateway_task_attempts model_attempt
LEFT JOIN platform_models selected_model ON selected_model.id = model_attempt.platform_model_id
WHERE model_attempt.task_id = t.id
AND (
COALESCE(selected_model.model_name, '') = $%d
OR COALESCE(selected_model.provider_model_name, '') = $%d
OR COALESCE(selected_model.model_alias, '') = $%d
)
)
)`, value)
}
if value := strings.TrimSpace(filter.ModelType); value != "" {
add("COALESCE(t.model_type, '') = $%d", value)
}
if value := strings.TrimSpace(filter.RunMode); value != "" {
add("t.run_mode = $%d", value)
}
if value := strings.TrimSpace(filter.BillingStatus); value != "" {
add("t.billing_status = $%d", value)
}
if value := strings.TrimSpace(filter.APIKey); value != "" {
add(`(
COALESCE(t.api_key_id, '') ILIKE $%d
OR COALESCE(t.api_key_name, '') ILIKE $%d
OR COALESCE(t.api_key_prefix, '') ILIKE $%d
)`, "%"+value+"%")
}
if filter.CreatedFrom != nil {
add("t.created_at >= $%d::timestamptz", *filter.CreatedFrom)
}
if filter.CreatedTo != nil {
add("t.created_at <= $%d::timestamptz", *filter.CreatedTo)
}
return where, args
}
func (s *Store) loadAdminTasksByIDs(ctx context.Context, taskIDs []string) ([]AdminGatewayTask, error) {
if len(taskIDs) == 0 {
return []AdminGatewayTask{}, nil
}
rows, err := s.pool.Query(ctx, `
SELECT `+gatewayTaskColumns+`
FROM gateway_tasks
WHERE id::text = ANY($1)`, taskIDs)
if err != nil {
return nil, err
}
defer rows.Close()
tasksByID := make(map[string]GatewayTask, len(taskIDs))
tasks := make([]GatewayTask, 0, len(taskIDs))
for rows.Next() {
task, err := scanGatewayTask(rows)
if err != nil {
return nil, err
}
tasks = append(tasks, task)
}
if err := rows.Err(); err != nil {
return nil, err
}
tasks, err = s.attachTaskAttempts(ctx, tasks)
if err != nil {
return nil, err
}
for _, task := range tasks {
tasksByID[task.ID] = task
}
contexts, err := s.loadAdminTaskContexts(ctx, taskIDs)
if err != nil {
return nil, err
}
items := make([]AdminGatewayTask, 0, len(taskIDs))
for _, taskID := range taskIDs {
task, ok := tasksByID[taskID]
if !ok {
continue
}
items = append(items, adminTaskWithContext(task, contexts[taskID]))
}
return items, nil
}
func (s *Store) loadAdminTaskContexts(ctx context.Context, taskIDs []string) (map[string]AdminTaskContext, error) {
contexts := make(map[string]AdminTaskContext, len(taskIDs))
if len(taskIDs) == 0 {
return contexts, nil
}
rows, err := s.pool.Query(ctx, `
SELECT t.id::text,
COALESCE(u.id::text, ''), COALESCE(u.username, ''), COALESCE(u.display_name, ''),
COALESCE(u.email, ''), COALESCE(u.source, ''),
COALESCE(tenant.id::text, ''), COALESCE(tenant.tenant_key, ''), COALESCE(tenant.name, '')
FROM gateway_tasks t
LEFT JOIN gateway_users u ON u.id = t.gateway_user_id
LEFT JOIN gateway_tenants tenant ON tenant.id = t.gateway_tenant_id
WHERE t.id::text = ANY($1)`, taskIDs)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var taskID string
var user AdminTaskUserSummary
var tenant AdminTaskTenantSummary
if err := rows.Scan(
&taskID,
&user.ID,
&user.Username,
&user.DisplayName,
&user.Email,
&user.Source,
&tenant.ID,
&tenant.TenantKey,
&tenant.Name,
); err != nil {
return nil, err
}
context := AdminTaskContext{}
if user.ID != "" {
context.User = &user
}
if tenant.ID != "" {
context.Tenant = &tenant
}
contexts[taskID] = context
}
return contexts, rows.Err()
}
func adminTaskWithContext(task GatewayTask, context AdminTaskContext) AdminGatewayTask {
if context.User == nil {
userID := strings.TrimSpace(task.GatewayUserID)
if userID == "" {
userID = strings.TrimSpace(task.UserID)
}
if userID != "" {
context.User = &AdminTaskUserSummary{ID: userID, Source: task.UserSource}
}
}
if context.Tenant == nil {
tenantID := strings.TrimSpace(task.GatewayTenantID)
if tenantID == "" {
tenantID = strings.TrimSpace(task.TenantID)
}
if tenantID != "" || strings.TrimSpace(task.TenantKey) != "" {
context.Tenant = &AdminTaskTenantSummary{ID: tenantID, TenantKey: task.TenantKey}
}
}
for index := len(task.Attempts) - 1; index >= 0; index-- {
attempt := task.Attempts[index]
if strings.TrimSpace(attempt.PlatformID) == "" {
continue
}
context.LatestPlatform = &AdminTaskPlatformSummary{
ID: attempt.PlatformID,
Name: attempt.PlatformName,
Provider: attempt.Provider,
}
break
}
return AdminGatewayTask{GatewayTask: task, AdminContext: context}
}