BCAI/easyai-ai-gateway
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(billing): 完成计费正确性 v2 闭环 (#10)
ci / verify (push) Failing after 3m44s

Browse Source 
统一计价、冻结、任务执行、异步结算、人工复核与前端即时估价闭环。

合并前已通过精确 ci / verify (pull_request)，包含 PostgreSQL 16 迁移与计费集成测试。
This commit was merged in pull request #10.
This commit is contained in:
chengcheng
2026-07-21 11:00:19 +08:00
parent dcf5c4f340 0aa9b3e88f
commit 1e55f7df8b
59 changed files with 6379 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+6
View File
@@ -23,6 +23,12 @@ CONFIG_JWT_SECRET=this is a very secret secret
# - hybrid: both sources are accepted and separated by gateway_users.source. # - hybrid: both sources are accepted and separated by gateway_users.source.
IDENTITY_MODE=hybrid IDENTITY_MODE=hybrid
# Billing engine rollout mode:
# - observe: keep legacy billing decisions and compare effective-pricing-v2 in logs.
# - enforce: require v2 pricing, reserve the candidate maximum, then settle asynchronously.
# - hold: reject new production generation before any upstream request; existing settlements continue.
BILLING_ENGINE_MODE=observe
# Unified identity business settings are managed in System Settings > Unified # Unified identity business settings are managed in System Settings > Unified
# Identity. Deployment only supplies the SecretStore and infrastructure timing. # Identity. Deployment only supplies the SecretStore and infrastructure timing.
AI_GATEWAY_PUBLIC_BASE_URL=http://localhost:8088 AI_GATEWAY_PUBLIC_BASE_URL=http://localhost:8088
.gitea/workflows/ci.yml
+17
View File
@@ -9,8 +9,22 @@ on:
jobs: jobs:
verify: verify:
runs-on: easyai-gateway-ci-unprivileged-v2 runs-on: easyai-gateway-ci-unprivileged-v2
services:
postgres:
image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
env:
POSTGRES_USER: easyai_test
POSTGRES_HOST_AUTH_METHOD: trust
POSTGRES_DB: easyai_gateway_test
options: >-
--health-cmd "pg_isready -U easyai_test -d easyai_gateway_test"
--health-interval 2s
--health-timeout 5s
--health-retries 30
env: env:
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
steps: steps:
- name: Checkout without external Actions - name: Checkout without external Actions
env: env:
@@ -79,6 +93,9 @@ jobs:
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
exit 1 exit 1
} }
- name: Migrate PostgreSQL 16 integration database
working-directory: apps/api
run: go run ./cmd/migrate
- name: Verify Go code - name: Verify Go code
working-directory: apps/api working-directory: apps/api
env: env:
.gitea/workflows/release-ci.yml
+17
View File
@@ -7,8 +7,22 @@ on:
jobs: jobs:
verify-tag: verify-tag:
runs-on: easyai-gateway-ci-unprivileged-v2 runs-on: easyai-gateway-ci-unprivileged-v2
services:
postgres:
image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777
env:
POSTGRES_USER: easyai_test
POSTGRES_HOST_AUTH_METHOD: trust
POSTGRES_DB: easyai_gateway_test
options: >-
--health-cmd "pg_isready -U easyai_test -d easyai_gateway_test"
--health-interval 2s
--health-timeout 5s
--health-retries 30
env: env:
TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2
AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable
steps: steps:
- name: Checkout without external Actions - name: Checkout without external Actions
env: env:
@@ -64,6 +78,9 @@ jobs:
printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2
exit 1 exit 1
} }
- name: Migrate PostgreSQL 16 integration database
working-directory: apps/api
run: go run ./cmd/migrate
- name: Verify Go code - name: Verify Go code
working-directory: apps/api working-directory: apps/api
env: env:
apps/api/docs/swagger.json
+489 -1
View File
@@ -1963,6 +1963,160 @@
} }
} }
}, },
"/api/admin/runtime/billing-settlements": {
"get": {
"security": [
{
"BearerAuth": []
}
],
"description": "管理端分页查询待结算、重试失败和人工复核记录。",
"produces": [
"application/json"
],
"tags": [
"billing"
],
"summary": "查询计费结算队列",
"parameters": [
{
"type": "string",
"description": "结算状态",
"name": "status",
"in": "query"
},
{
"type": "string",
"description": "动作：settle 或 release",
"name": "action",
"in": "query"
},
{
"type": "integer",
"default": 1,
"description": "页码",
"name": "page",
"in": "query"
},
{
"type": "integer",
"default": 50,
"description": "每页数量",
"name": "pageSize",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/httpapi.BillingSettlementListResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
}
}
}
},
"/api/admin/runtime/billing-settlements/{settlementId}/retry": {
"post": {
"security": [
{
"BearerAuth": []
}
],
"description": "Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。",
"produces": [
"application/json"
],
"tags": [
"billing"
],
"summary": "重试计费结算",
"parameters": [
{
"type": "string",
"description": "结算记录 ID",
"name": "settlementId",
"in": "path",
"required": true
},
{
"type": "string",
"description": "幂等键",
"name": "Idempotency-Key",
"in": "header",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/httpapi.BillingSettlementRetryResponse"
}
},
"400": {
"description": "Bad Request",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"401": {
"description": "Unauthorized",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"403": {
"description": "Forbidden",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"409": {
"description": "Conflict",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
}
}
}
},
"/api/admin/runtime/model-rate-limits": { "/api/admin/runtime/model-rate-limits": {
"get": { "get": {
"security": [ "security": [
@@ -5340,6 +5494,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -5501,6 +5661,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -5594,6 +5760,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -5804,6 +5976,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -6017,6 +6195,12 @@
"schema": { "schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope" "$ref": "#/definitions/httpapi.ErrorEnvelope"
} }
},
"503": {
"description": "Service Unavailable",
"schema": {
"$ref": "#/definitions/httpapi.ErrorEnvelope"
}
} }
} }
} }
@@ -6196,6 +6380,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -6439,6 +6629,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -6532,6 +6728,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -6934,6 +7136,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -7027,6 +7235,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -7847,6 +8061,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -7960,6 +8180,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -8053,6 +8279,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -8146,6 +8378,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -8265,6 +8503,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -8434,6 +8678,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -8527,6 +8777,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9112,6 +9368,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9273,6 +9535,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9366,6 +9634,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9459,6 +9733,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9552,6 +9832,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9721,6 +10007,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9814,6 +10106,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -9965,6 +10263,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -10161,6 +10465,12 @@
"name": "X-Async", "name": "X-Async",
"in": "header" "in": "header"
}, },
{
"type": "string",
"description": "可选请求幂等键；同一用户范围内唯一",
"name": "Idempotency-Key",
"in": "header"
},
{ {
"description": "AI 任务请求，字段随任务类型变化", "description": "AI 任务请求，字段随任务类型变化",
"name": "input", "name": "input",
@@ -10463,6 +10773,40 @@
} }
} }
}, },
"httpapi.BillingSettlementListResponse": {
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {
"$ref": "#/definitions/store.BillingSettlement"
}
},
"page": {
"type": "integer",
"example": 1
},
"pageSize": {
"type": "integer",
"example": 50
},
"total": {
"type": "integer",
"example": 42
}
}
},
"httpapi.BillingSettlementRetryResponse": {
"type": "object",
"properties": {
"auditLog": {
"$ref": "#/definitions/store.AuditLog"
},
"settlement": {
"$ref": "#/definitions/store.BillingSettlement"
}
}
},
"httpapi.CatalogProviderListResponse": { "httpapi.CatalogProviderListResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -11227,6 +11571,14 @@
"httpapi.PricingEstimateResponse": { "httpapi.PricingEstimateResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
"candidateCount": {
"type": "integer",
"example": 2
},
"currency": {
"type": "string",
"example": "resource"
},
"items": { "items": {
"type": "array", "type": "array",
"items": { "items": {
@@ -11234,9 +11586,25 @@
"additionalProperties": true "additionalProperties": true
} }
}, },
"pricingVersion": {
"type": "string",
"example": "effective-pricing-v2"
},
"requestFingerprint": {
"type": "string",
"example": "76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f"
},
"reservationAmount": {
"type": "number",
"example": 2.75
},
"resolver": { "resolver": {
"type": "string", "type": "string",
"example": "effective-pricing-v1" "example": "effective-pricing-v2"
},
"totalAmount": {
"type": "number",
"example": 1.25
} }
} }
}, },
@@ -12628,6 +12996,67 @@
} }
} }
}, },
"store.BillingSettlement": {
"type": "object",
"properties": {
"action": {
"type": "string"
},
"amount": {
"type": "number"
},
"attempts": {
"type": "integer"
},
"completedAt": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"currency": {
"type": "string"
},
"id": {
"type": "string"
},
"lastErrorCode": {
"type": "string"
},
"lastErrorMessage": {
"type": "string"
},
"lockedAt": {
"type": "string"
},
"lockedBy": {
"type": "string"
},
"manualReviewReason": {
"type": "string"
},
"nextAttemptAt": {
"type": "string"
},
"payload": {
"type": "object",
"additionalProperties": {}
},
"pricingSnapshot": {
"type": "object",
"additionalProperties": {}
},
"status": {
"type": "string"
},
"taskId": {
"type": "string"
},
"updatedAt": {
"type": "string"
}
}
},
"store.CatalogProvider": { "store.CatalogProvider": {
"type": "object", "type": "object",
"properties": { "properties": {
@@ -13059,10 +13488,25 @@
"$ref": "#/definitions/store.TaskAttempt" "$ref": "#/definitions/store.TaskAttempt"
} }
}, },
"billingCurrency": {
"type": "string"
},
"billingSettledAt": {
"type": "string"
},
"billingStatus": {
"type": "string"
},
"billingSummary": { "billingSummary": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
}, },
"billingUpdatedAt": {
"type": "string"
},
"billingVersion": {
"type": "string"
},
"billings": { "billings": {
"type": "array", "type": "array",
"items": {} "items": {}
@@ -13085,6 +13529,9 @@
"errorMessage": { "errorMessage": {
"type": "string" "type": "string"
}, },
"executionLeaseExpiresAt": {
"type": "string"
},
"finalChargeAmount": { "finalChargeAmount": {
"type": "number" "type": "number"
}, },
@@ -13119,6 +13566,10 @@
"newMessageCount": { "newMessageCount": {
"type": "integer" "type": "integer"
}, },
"pricingSnapshot": {
"type": "object",
"additionalProperties": {}
},
"remoteTaskId": { "remoteTaskId": {
"type": "string" "type": "string"
}, },
@@ -13130,12 +13581,18 @@
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
}, },
"requestFingerprint": {
"type": "string"
},
"requestId": { "requestId": {
"type": "string" "type": "string"
}, },
"requestedModel": { "requestedModel": {
"type": "string" "type": "string"
}, },
"reservationAmount": {
"type": "number"
},
"resolvedModel": { "resolvedModel": {
"type": "string" "type": "string"
}, },
@@ -13966,6 +14423,12 @@
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
}, },
"effectiveFrom": {
"type": "string"
},
"effectiveTo": {
"type": "string"
},
"formulaConfig": { "formulaConfig": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
@@ -13973,6 +14436,9 @@
"id": { "id": {
"type": "string" "type": "string"
}, },
"isFree": {
"type": "boolean"
},
"metadata": { "metadata": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
@@ -14033,10 +14499,19 @@
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
}, },
"effectiveFrom": {
"type": "string"
},
"effectiveTo": {
"type": "string"
},
"formulaConfig": { "formulaConfig": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
}, },
"isFree": {
"type": "boolean"
},
"metadata": { "metadata": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
@@ -14458,6 +14933,10 @@
"platformName": { "platformName": {
"type": "string" "type": "string"
}, },
"pricingSnapshot": {
"type": "object",
"additionalProperties": {}
},
"provider": { "provider": {
"type": "string" "type": "string"
}, },
@@ -14467,6 +14946,9 @@
"queueKey": { "queueKey": {
"type": "string" "type": "string"
}, },
"requestFingerprint": {
"type": "string"
},
"requestId": { "requestId": {
"type": "string" "type": "string"
}, },
@@ -14505,6 +14987,12 @@
"taskId": { "taskId": {
"type": "string" "type": "string"
}, },
"upstreamSubmissionStatus": {
"type": "string"
},
"upstreamSubmissionUpdatedAt": {
"type": "string"
},
"usage": { "usage": {
"type": "object", "type": "object",
"additionalProperties": {} "additionalProperties": {}
apps/api/docs/swagger.yaml
+328 -2
View File
@@ -85,6 +85,29 @@ definitions:
$ref: '#/definitions/store.BaseModel' $ref: '#/definitions/store.BaseModel'
type: array type: array
type: object type: object
httpapi.BillingSettlementListResponse:
properties:
items:
items:
$ref: '#/definitions/store.BillingSettlement'
type: array
page:
example: 1
type: integer
pageSize:
example: 50
type: integer
total:
example: 42
type: integer
type: object
httpapi.BillingSettlementRetryResponse:
properties:
auditLog:
$ref: '#/definitions/store.AuditLog'
settlement:
$ref: '#/definitions/store.BillingSettlement'
type: object
httpapi.CatalogProviderListResponse: httpapi.CatalogProviderListResponse:
properties: properties:
items: items:
@@ -610,14 +633,32 @@ definitions:
type: object type: object
httpapi.PricingEstimateResponse: httpapi.PricingEstimateResponse:
properties: properties:
candidateCount:
example: 2
type: integer
currency:
example: resource
type: string
items: items:
items: items:
additionalProperties: true additionalProperties: true
type: object type: object
type: array type: array
resolver: pricingVersion:
example: effective-pricing-v1 example: effective-pricing-v2
type: string type: string
requestFingerprint:
example: 76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f
type: string
reservationAmount:
example: 2.75
type: number
resolver:
example: effective-pricing-v2
type: string
totalAmount:
example: 1.25
type: number
type: object type: object
httpapi.PricingRuleListResponse: httpapi.PricingRuleListResponse:
properties: properties:
@@ -1581,6 +1622,47 @@ definitions:
status: status:
type: string type: string
type: object type: object
store.BillingSettlement:
properties:
action:
type: string
amount:
type: number
attempts:
type: integer
completedAt:
type: string
createdAt:
type: string
currency:
type: string
id:
type: string
lastErrorCode:
type: string
lastErrorMessage:
type: string
lockedAt:
type: string
lockedBy:
type: string
manualReviewReason:
type: string
nextAttemptAt:
type: string
payload:
additionalProperties: {}
type: object
pricingSnapshot:
additionalProperties: {}
type: object
status:
type: string
taskId:
type: string
updatedAt:
type: string
type: object
store.CatalogProvider: store.CatalogProvider:
properties: properties:
capabilitySchema: capabilitySchema:
@@ -1872,9 +1954,19 @@ definitions:
items: items:
$ref: '#/definitions/store.TaskAttempt' $ref: '#/definitions/store.TaskAttempt'
type: array type: array
billingCurrency:
type: string
billingSettledAt:
type: string
billingStatus:
type: string
billingSummary: billingSummary:
additionalProperties: {} additionalProperties: {}
type: object type: object
billingUpdatedAt:
type: string
billingVersion:
type: string
billings: billings:
items: {} items: {}
type: array type: array
@@ -1890,6 +1982,8 @@ definitions:
type: string type: string
errorMessage: errorMessage:
type: string type: string
executionLeaseExpiresAt:
type: string
finalChargeAmount: finalChargeAmount:
type: number type: number
finishedAt: finishedAt:
@@ -1913,6 +2007,9 @@ definitions:
type: string type: string
newMessageCount: newMessageCount:
type: integer type: integer
pricingSnapshot:
additionalProperties: {}
type: object
remoteTaskId: remoteTaskId:
type: string type: string
remoteTaskPayload: remoteTaskPayload:
@@ -1921,10 +2018,14 @@ definitions:
request: request:
additionalProperties: {} additionalProperties: {}
type: object type: object
requestFingerprint:
type: string
requestId: requestId:
type: string type: string
requestedModel: requestedModel:
type: string type: string
reservationAmount:
type: number
resolvedModel: resolvedModel:
type: string type: string
responseDurationMs: responseDurationMs:
@@ -2485,11 +2586,17 @@ definitions:
dynamicWeight: dynamicWeight:
additionalProperties: {} additionalProperties: {}
type: object type: object
effectiveFrom:
type: string
effectiveTo:
type: string
formulaConfig: formulaConfig:
additionalProperties: {} additionalProperties: {}
type: object type: object
id: id:
type: string type: string
isFree:
type: boolean
metadata: metadata:
additionalProperties: {} additionalProperties: {}
type: object type: object
@@ -2531,9 +2638,15 @@ definitions:
dynamicWeight: dynamicWeight:
additionalProperties: {} additionalProperties: {}
type: object type: object
effectiveFrom:
type: string
effectiveTo:
type: string
formulaConfig: formulaConfig:
additionalProperties: {} additionalProperties: {}
type: object type: object
isFree:
type: boolean
metadata: metadata:
additionalProperties: {} additionalProperties: {}
type: object type: object
@@ -2820,12 +2933,17 @@ definitions:
type: string type: string
platformName: platformName:
type: string type: string
pricingSnapshot:
additionalProperties: {}
type: object
provider: provider:
type: string type: string
providerModelName: providerModelName:
type: string type: string
queueKey: queueKey:
type: string type: string
requestFingerprint:
type: string
requestId: requestId:
type: string type: string
requestSnapshot: requestSnapshot:
@@ -2852,6 +2970,10 @@ definitions:
type: integer type: integer
taskId: taskId:
type: string type: string
upstreamSubmissionStatus:
type: string
upstreamSubmissionUpdatedAt:
type: string
usage: usage:
additionalProperties: {} additionalProperties: {}
type: object type: object
@@ -4234,6 +4356,106 @@ paths:
summary: 列出定价规则 summary: 列出定价规则
tags: tags:
- pricing - pricing
/api/admin/runtime/billing-settlements:
get:
description: 管理端分页查询待结算、重试失败和人工复核记录。
parameters:
- description: 结算状态
in: query
name: status
type: string
- description: 动作：settle 或 release
in: query
name: action
type: string
- default: 1
description: 页码
in: query
name: page
type: integer
- default: 50
description: 每页数量
in: query
name: pageSize
type: integer
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/httpapi.BillingSettlementListResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"403":
description: Forbidden
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
security:
- BearerAuth: []
summary: 查询计费结算队列
tags:
- billing
/api/admin/runtime/billing-settlements/{settlementId}/retry:
post:
description: Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。
parameters:
- description: 结算记录 ID
in: path
name: settlementId
required: true
type: string
- description: 幂等键
in: header
name: Idempotency-Key
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/httpapi.BillingSettlementRetryResponse'
"400":
description: Bad Request
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"401":
description: Unauthorized
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"403":
description: Forbidden
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"404":
description: Not Found
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"409":
description: Conflict
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
security:
- BearerAuth: []
summary: 重试计费结算
tags:
- billing
/api/admin/runtime/model-rate-limits: /api/admin/runtime/model-rate-limits:
get: get:
description: 管理端查看平台模型维度的限流和冷却状态。 description: 管理端查看平台模型维度的限流和冷却状态。
@@ -6402,6 +6624,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -6507,6 +6733,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -6568,6 +6798,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -6702,6 +6936,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -6841,6 +7079,10 @@ paths:
description: Internal Server Error description: Internal Server Error
schema: schema:
$ref: '#/definitions/httpapi.ErrorEnvelope' $ref: '#/definitions/httpapi.ErrorEnvelope'
"503":
description: Service Unavailable
schema:
$ref: '#/definitions/httpapi.ErrorEnvelope'
security: security:
- BearerAuth: [] - BearerAuth: []
summary: 估算请求价格 summary: 估算请求价格
@@ -6955,6 +7197,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -7119,6 +7365,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -7180,6 +7430,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -7439,6 +7693,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -7500,6 +7758,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8028,6 +8290,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8102,6 +8368,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8163,6 +8433,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8224,6 +8498,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8302,6 +8580,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8415,6 +8697,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8476,6 +8762,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8856,6 +9146,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -8961,6 +9255,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9022,6 +9320,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9083,6 +9385,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9144,6 +9450,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9257,6 +9567,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9318,6 +9632,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9416,6 +9734,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
@@ -9543,6 +9865,10 @@ paths:
in: header in: header
name: X-Async name: X-Async
type: boolean type: boolean
- description: 可选请求幂等键；同一用户范围内唯一
in: header
name: Idempotency-Key
type: string
- description: AI 任务请求，字段随任务类型变化 - description: AI 任务请求，字段随任务类型变化
in: body in: body
name: input name: input
apps/api/internal/clients/openai_request_params.go
+5
View File
@@ -34,6 +34,11 @@ var gatewayOpenAIRequestExtensions = stringSet(
"requestId", "request_id", "signal", "userMessage", "user_message", "platformId", "requestId", "request_id", "signal", "userMessage", "user_message", "platformId",
"platform_id", "options", "enable_thinking", "thinking_budget_tokens", "enable_web_search", "platform_id", "options", "enable_thinking", "thinking_budget_tokens", "enable_web_search",
"modelType", "model_type", "capability", "capabilityType", "mode", "simulation", "testMode", "modelType", "model_type", "capability", "capabilityType", "mode", "simulation", "testMode",
"cacheAffinityKey", "cache_affinity_key", "simulationDurationMs", "simulationDurationSeconds",
"simulationMinDurationMs", "simulationMaxDurationMs", "simulationMinDurationSeconds",
"simulationMaxDurationSeconds", "simulationDurationMinMs", "simulationDurationMaxMs",
"simulationDurationMinSeconds", "simulationDurationMaxSeconds", "simulationFailure",
"simulationProfile", "simulationUsage",
) )
var gatewayResponsesRequestExtensions = stringSet("messages", "presence_penalty", "frequency_penalty") var gatewayResponsesRequestExtensions = stringSet("messages", "presence_penalty", "frequency_penalty")
apps/api/internal/clients/openai_request_params_test.go
+17
View File
@@ -60,6 +60,23 @@ func TestValidateOpenAIRequestParametersRejectsUnknownTopLevelField(t *testing.T
} }
} }
func TestValidateOpenAIRequestParametersAcceptsInternalSimulationAndAffinityFields(t *testing.T) {
body := map[string]any{
"model": "demo", "messages": []any{}, "simulation": true,
"simulationDurationMs": 5, "simulationUsage": map[string]any{"inputTokens": 10},
"cacheAffinityKey": "conversation-1",
}
if err := ValidateOpenAIRequestParameters("chat.completions", body); err != nil {
t.Fatalf("expected documented gateway extensions to remain accepted, got %v", err)
}
filtered := FilterOpenAIChatRequestBody(body)
for _, key := range []string{"simulation", "simulationDurationMs", "simulationUsage", "cacheAffinityKey"} {
if _, ok := filtered[key]; ok {
t.Fatalf("gateway-only field %q leaked upstream", key)
}
}
}
func TestResponsesFallbackMapsEquivalentCurrentParameters(t *testing.T) { func TestResponsesFallbackMapsEquivalentCurrentParameters(t *testing.T) {
body, err := ResponsesRequestToChat(map[string]any{ body, err := ResponsesRequestToChat(map[string]any{
"input": "hello", "store": false, "metadata": map[string]any{"trace": "1"}, "input": "hello", "store": false, "metadata": map[string]any{"trace": "1"},
apps/api/internal/clients/types.go
+1
View File
@@ -36,6 +36,7 @@ type ResponseTurn struct {
} }
type Response struct { type Response struct {
AttemptID string
Result map[string]any Result map[string]any
RequestID string RequestID string
Usage Usage Usage Usage
apps/api/internal/config/config.go
+7
View File
@@ -47,6 +47,7 @@ type Config struct {
GlobalHTTPProxy string GlobalHTTPProxy string
GlobalHTTPProxySource string GlobalHTTPProxySource string
LogLevel slog.Level LogLevel slog.Level
BillingEngineMode string
} }
func Load() Config { func Load() Config {
@@ -90,10 +91,16 @@ func Load() Config {
GlobalHTTPProxy: globalProxy.HTTPProxy, GlobalHTTPProxy: globalProxy.HTTPProxy,
GlobalHTTPProxySource: globalProxy.Source, GlobalHTTPProxySource: globalProxy.Source,
LogLevel: logLevel(env("LOG_LEVEL", "info")), LogLevel: logLevel(env("LOG_LEVEL", "info")),
BillingEngineMode: strings.ToLower(env("BILLING_ENGINE_MODE", "observe")),
} }
} }
func (c Config) Validate() error { func (c Config) Validate() error {
switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) {
case "", "observe", "enforce", "hold":
default:
return errors.New("BILLING_ENGINE_MODE must be observe, enforce, or hold")
}
switch strings.ToLower(strings.TrimSpace(c.IdentitySecretStore)) { switch strings.ToLower(strings.TrimSpace(c.IdentitySecretStore)) {
case "": case "":
case "file": case "file":
apps/api/internal/httpapi/billing_admin_handlers.go
+10 -12
View File
@@ -13,7 +13,7 @@ import (
type walletBalanceRequest struct { type walletBalanceRequest struct {
Currency string `json:"currency" example:"USD"` Currency string `json:"currency" example:"USD"`
Balance float64 `json:"balance" example:"100"` Balance json.Number `json:"balance" swaggertype:"number" example:"100"`
Reason string `json:"reason" example:"manual recharge"` Reason string `json:"reason" example:"manual recharge"`
IdempotencyKey string `json:"idempotencyKey" example:"wallet-set-20260514-001"` IdempotencyKey string `json:"idempotencyKey" example:"wallet-set-20260514-001"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
@@ -21,7 +21,7 @@ type walletBalanceRequest struct {
type walletRechargeRequest struct { type walletRechargeRequest struct {
Currency string `json:"currency" example:"resource"` Currency string `json:"currency" example:"resource"`
Amount float64 `json:"amount" example:"100"` Amount json.Number `json:"amount" swaggertype:"number" example:"100"`
Reason string `json:"reason" example:"manual recharge"` Reason string `json:"reason" example:"manual recharge"`
IdempotencyKey string `json:"idempotencyKey" example:"wallet-recharge-20260514-001"` IdempotencyKey string `json:"idempotencyKey" example:"wallet-recharge-20260514-001"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
@@ -50,10 +50,6 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, "invalid json body") writeError(w, http.StatusBadRequest, "invalid json body")
return return
} }
if input.Balance < 0 {
writeError(w, http.StatusBadRequest, "wallet balance cannot be negative")
return
}
gatewayUserID := strings.TrimSpace(r.PathValue("userID")) gatewayUserID := strings.TrimSpace(r.PathValue("userID"))
reason := strings.TrimSpace(input.Reason) reason := strings.TrimSpace(input.Reason)
if reason == "" { if reason == "" {
@@ -67,7 +63,7 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) {
next, err := s.store.SetUserWalletBalanceTx(r.Context(), tx, store.WalletBalanceAdjustmentInput{ next, err := s.store.SetUserWalletBalanceTx(r.Context(), tx, store.WalletBalanceAdjustmentInput{
GatewayUserID: gatewayUserID, GatewayUserID: gatewayUserID,
Currency: input.Currency, Currency: input.Currency,
Balance: input.Balance, BalanceText: input.Balance.String(),
Reason: reason, Reason: reason,
IdempotencyKey: input.IdempotencyKey, IdempotencyKey: input.IdempotencyKey,
Metadata: input.Metadata, Metadata: input.Metadata,
@@ -89,6 +85,10 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusNotFound, "user not found") writeError(w, http.StatusNotFound, "user not found")
case errors.Is(err, store.ErrWalletBalanceUnchanged): case errors.Is(err, store.ErrWalletBalanceUnchanged):
writeError(w, http.StatusBadRequest, "wallet balance is unchanged") writeError(w, http.StatusBadRequest, "wallet balance is unchanged")
case errors.Is(err, store.ErrInvalidWalletAmount):
writeError(w, http.StatusBadRequest, "wallet balance must be a non-negative decimal with at most nine fractional digits", "invalid_wallet_amount")
case errors.Is(err, store.ErrBalanceBelowFrozen):
writeError(w, http.StatusConflict, "wallet balance cannot be below frozen balance", "balance_below_frozen")
default: default:
s.logger.Error("set user wallet balance failed", "error", err) s.logger.Error("set user wallet balance failed", "error", err)
writeError(w, http.StatusInternalServerError, "set user wallet balance failed") writeError(w, http.StatusInternalServerError, "set user wallet balance failed")
@@ -126,10 +126,6 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques
writeError(w, http.StatusBadRequest, "invalid json body") writeError(w, http.StatusBadRequest, "invalid json body")
return return
} }
if input.Amount <= 0 {
writeError(w, http.StatusBadRequest, "wallet recharge amount must be positive")
return
}
gatewayUserID := strings.TrimSpace(r.PathValue("userID")) gatewayUserID := strings.TrimSpace(r.PathValue("userID"))
reason := strings.TrimSpace(input.Reason) reason := strings.TrimSpace(input.Reason)
if reason == "" { if reason == "" {
@@ -143,7 +139,7 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques
next, err := s.store.RechargeUserWalletBalanceTx(r.Context(), tx, store.WalletRechargeInput{ next, err := s.store.RechargeUserWalletBalanceTx(r.Context(), tx, store.WalletRechargeInput{
GatewayUserID: gatewayUserID, GatewayUserID: gatewayUserID,
Currency: input.Currency, Currency: input.Currency,
Amount: input.Amount, AmountText: input.Amount.String(),
Reason: reason, Reason: reason,
IdempotencyKey: input.IdempotencyKey, IdempotencyKey: input.IdempotencyKey,
Metadata: input.Metadata, Metadata: input.Metadata,
@@ -163,6 +159,8 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques
switch { switch {
case store.IsNotFound(err): case store.IsNotFound(err):
writeError(w, http.StatusNotFound, "user not found") writeError(w, http.StatusNotFound, "user not found")
case errors.Is(err, store.ErrInvalidWalletAmount):
writeError(w, http.StatusBadRequest, "wallet recharge amount must be a positive decimal with at most nine fractional digits", "invalid_wallet_amount")
default: default:
s.logger.Error("recharge user wallet balance failed", "error", err) s.logger.Error("recharge user wallet balance failed", "error", err)
writeError(w, http.StatusInternalServerError, "recharge user wallet balance failed") writeError(w, http.StatusInternalServerError, "recharge user wallet balance failed")
apps/api/internal/httpapi/billing_settlement_handlers.go
+145
View File
@@ -0,0 +1,145 @@
package httpapi
import (
"crypto/sha256"
"encoding/hex"
"errors"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
// listBillingSettlements godoc
// @Summary 查询计费结算队列
// @Description 管理端分页查询待结算、重试失败和人工复核记录。
// @Tags billing
// @Produce json
// @Security BearerAuth
// @Param status query string false "结算状态"
// @Param action query string false "动作：settle 或 release"
// @Param page query int false "页码" default(1)
// @Param pageSize query int false "每页数量" default(50)
// @Success 200 {object} BillingSettlementListResponse
// @Failure 400 {object} ErrorEnvelope
// @Failure 401 {object} ErrorEnvelope
// @Failure 403 {object} ErrorEnvelope
// @Failure 500 {object} ErrorEnvelope
// @Router /api/admin/runtime/billing-settlements [get]
func (s *Server) listBillingSettlements(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
page, err := positiveQueryInt(query.Get("page"), 1)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid page", "invalid_request")
return
}
pageSize, err := positiveQueryInt(query.Get("pageSize"), 50)
if err != nil {
writeError(w, http.StatusBadRequest, "invalid pageSize", "invalid_request")
return
}
result, err := s.store.ListBillingSettlements(r.Context(), store.BillingSettlementListFilter{
Status: query.Get("status"), Action: query.Get("action"), Page: page, PageSize: pageSize,
})
if err != nil {
s.logger.Error("list billing settlements failed", "error_category", "billing_settlement_list_failed")
writeError(w, http.StatusInternalServerError, "list billing settlements failed", "billing_settlement_list_failed")
return
}
writeJSON(w, http.StatusOK, result)
}
// retryBillingSettlement godoc
// @Summary 重试计费结算
// @Description Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。
// @Tags billing
// @Produce json
// @Security BearerAuth
// @Param settlementId path string true "结算记录 ID"
// @Param Idempotency-Key header string true "幂等键"
// @Success 200 {object} BillingSettlementRetryResponse
// @Failure 400 {object} ErrorEnvelope
// @Failure 401 {object} ErrorEnvelope
// @Failure 403 {object} ErrorEnvelope
// @Failure 404 {object} ErrorEnvelope
// @Failure 409 {object} ErrorEnvelope
// @Failure 500 {object} ErrorEnvelope
// @Router /api/admin/runtime/billing-settlements/{settlementId}/retry [post]
func (s *Server) retryBillingSettlement(w http.ResponseWriter, r *http.Request) {
key, ok := singleIdempotencyKey(r)
if !ok {
writeError(w, http.StatusBadRequest, "a single Idempotency-Key is required", "idempotency_key_required")
return
}
keyHash := sha256.Sum256([]byte(key))
settlementID := strings.TrimSpace(r.PathValue("settlementId"))
if _, err := uuid.Parse(settlementID); err != nil {
writeError(w, http.StatusBadRequest, "invalid billing settlement ID", "invalid_request")
return
}
actor, _ := auth.UserFromContext(r.Context())
var item store.BillingSettlement
var audit store.AuditLog
var replayed bool
err := s.store.InTx(r.Context(), func(tx store.Tx) error {
var err error
item, replayed, err = s.store.RetryBillingSettlementTx(r.Context(), tx, settlementID, hex.EncodeToString(keyHash[:]))
if err != nil || replayed {
return err
}
audit, err = s.store.RecordAuditLogTx(r.Context(), tx, billingSettlementRetryAuditInput(r, actor, item))
return err
})
if err != nil {
switch {
case store.IsNotFound(err):
writeError(w, http.StatusNotFound, "billing settlement not found", "billing_settlement_not_found")
case errors.Is(err, store.ErrBillingSettlementNotRetryable):
writeError(w, http.StatusConflict, "billing settlement is not retryable", "billing_settlement_not_retryable")
default:
s.logger.Error("retry billing settlement failed", "settlementID", settlementID, "error_category", "billing_settlement_retry_failed")
writeError(w, http.StatusInternalServerError, "retry billing settlement failed", "billing_settlement_retry_failed")
}
return
}
if replayed {
w.Header().Set("Idempotent-Replayed", "true")
}
response := map[string]any{"settlement": item}
if !replayed {
response["auditLog"] = audit
}
writeJSON(w, http.StatusOK, response)
}
func singleIdempotencyKey(r *http.Request) (string, bool) {
values := r.Header.Values("Idempotency-Key")
if len(values) != 1 {
return "", false
}
value := strings.TrimSpace(values[0])
if value == "" || len(value) > 255 || strings.Contains(value, ",") {
return "", false
}
return value, true
}
func billingSettlementRetryAuditInput(r *http.Request, actor *auth.User, item store.BillingSettlement) store.AuditLogInput {
input := store.AuditLogInput{
Category: "billing", Action: "billing.settlement.retry",
TargetType: "billing_settlement", TargetID: item.ID,
RequestIP: requestIP(r), UserAgent: r.UserAgent(),
AfterState: map[string]any{"status": item.Status, "attempts": item.Attempts},
Metadata: map[string]any{"taskId": item.TaskID, "action": item.Action, "currency": item.Currency},
}
if actor != nil {
input.ActorGatewayUserID = uuidText(firstNonEmptyText(actor.GatewayUserID, actor.ID))
input.ActorUserID = actor.ID
input.ActorUsername = actor.Username
input.ActorSource = actor.Source
input.ActorRoles = actor.Roles
}
return input
}
apps/api/internal/httpapi/billing_settlement_handlers_test.go
+21
View File
@@ -0,0 +1,21 @@
package httpapi
import (
"net/http/httptest"
"testing"
)
func TestSingleIdempotencyKey(t *testing.T) {
t.Parallel()
request := httptest.NewRequest("POST", "/", nil)
request.Header.Add("Idempotency-Key", "retry-1")
if got, ok := singleIdempotencyKey(request); !ok || got != "retry-1" {
t.Fatalf("got key=%q ok=%v", got, ok)
}
request.Header.Add("Idempotency-Key", "retry-2")
if _, ok := singleIdempotencyKey(request); ok {
t.Fatal("multiple Idempotency-Key values must be rejected")
}
}
apps/api/internal/httpapi/core_flow_integration_test.go
+137 -14
View File
@@ -48,6 +48,7 @@ func TestCoreLocalFlow(t *testing.T) {
DatabaseURL: databaseURL, DatabaseURL: databaseURL,
IdentityMode: "hybrid", IdentityMode: "hybrid",
JWTSecret: "test-secret", JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true, TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress", TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*", CORSAllowedOrigin: "*",
@@ -444,7 +445,6 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
t.Fatalf("unexpected compatible chat response: %+v", compatChat) t.Fatalf("unexpected compatible chat response: %+v", compatChat)
} }
cancelMarker := "cancel-stream-" + suffixText
cancelCtx, cancelRequest := context.WithCancel(context.Background()) cancelCtx, cancelRequest := context.WithCancel(context.Background())
cancelPayload := map[string]any{ cancelPayload := map[string]any{
"model": defaultTextModel, "model": defaultTextModel,
@@ -453,7 +453,6 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
"stream": true, "stream": true,
"simulation": true, "simulation": true,
"simulationDurationMs": 250, "simulationDurationMs": 250,
"cancelTestId": cancelMarker,
} }
cancelRaw, err := json.Marshal(cancelPayload) cancelRaw, err := json.Marshal(cancelPayload)
if err != nil { if err != nil {
@@ -466,15 +465,27 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil {
cancelReq.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret) cancelReq.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret)
cancelReq.Header.Set("Content-Type", "application/json") cancelReq.Header.Set("Content-Type", "application/json")
cancelErrCh := make(chan error, 1) cancelErrCh := make(chan error, 1)
cancelTaskIDCh := make(chan string, 1)
go func() { go func() {
resp, err := http.DefaultClient.Do(cancelReq) resp, err := http.DefaultClient.Do(cancelReq)
if resp != nil { if resp != nil {
cancelTaskIDCh <- strings.TrimSpace(resp.Header.Get("X-Gateway-Task-Id"))
_, _ = io.ReadAll(resp.Body) _, _ = io.ReadAll(resp.Body)
_ = resp.Body.Close() _ = resp.Body.Close()
} else {
cancelTaskIDCh <- ""
} }
cancelErrCh <- err cancelErrCh <- err
}() }()
cancelTaskID := waitForTaskIDByRequestMarker(t, ctx, testPool, cancelMarker, 2*time.Second) var cancelTaskID string
select {
case cancelTaskID = <-cancelTaskIDCh:
case <-time.After(2 * time.Second):
t.Fatal("cancelled stream did not return response headers")
}
if cancelTaskID == "" {
t.Fatal("cancelled stream response did not expose X-Gateway-Task-Id")
}
cancelRequest() cancelRequest()
select { select {
case <-cancelErrCh: case <-cancelErrCh:
@@ -804,6 +815,7 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil {
"currency": "resource", "currency": "resource",
"rules": []map[string]any{ "rules": []map[string]any{
{"ruleKey": "text_input", "displayName": "Text Input", "resourceType": "text_input", "unit": "1k_tokens", "basePrice": 1}, {"ruleKey": "text_input", "displayName": "Text Input", "resourceType": "text_input", "unit": "1k_tokens", "basePrice": 1},
{"ruleKey": "text_cached_input", "displayName": "Text Cached Input", "resourceType": "text_cached_input", "unit": "1k_tokens", "basePrice": 0.5},
{"ruleKey": "text_output", "displayName": "Text Output", "resourceType": "text_output", "unit": "1k_tokens", "basePrice": 2}, {"ruleKey": "text_output", "displayName": "Text Output", "resourceType": "text_output", "unit": "1k_tokens", "basePrice": 2},
{"ruleKey": "image", "displayName": "Image", "resourceType": "image", "unit": "image", "basePrice": 7}, {"ruleKey": "image", "displayName": "Image", "resourceType": "image", "unit": "image", "basePrice": 7},
{"ruleKey": "image_edit", "displayName": "Image Edit", "resourceType": "image_edit", "unit": "image", "basePrice": 11}, {"ruleKey": "image_edit", "displayName": "Image Edit", "resourceType": "image_edit", "unit": "image", "basePrice": 11},
@@ -835,6 +847,101 @@ WHERE gateway_user_id = $1::uuid
AND currency = 'resource'`, smokeGatewayUserID).Scan(&walletBalanceBefore); err != nil { AND currency = 'resource'`, smokeGatewayUserID).Scan(&walletBalanceBefore); err != nil {
t.Fatalf("read wallet balance before pricing task: %v", err) t.Fatalf("read wallet balance before pricing task: %v", err)
} }
var estimateSideEffectsBefore struct {
Tasks int
Transactions int
Frozen string
}
if err := testPool.QueryRow(ctx, `
SELECT (SELECT count(*) FROM gateway_tasks),
(SELECT count(*) FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid),
(SELECT frozen_balance::text FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid AND currency='resource')`,
smokeGatewayUserID,
).Scan(&estimateSideEffectsBefore.Tasks, &estimateSideEffectsBefore.Transactions, &estimateSideEffectsBefore.Frozen); err != nil {
t.Fatalf("read estimate side effects before request: %v", err)
}
var pricingEstimate struct {
TotalAmount float64 `json:"totalAmount"`
ReservationAmount float64 `json:"reservationAmount"`
CandidateCount int `json:"candidateCount"`
PricingVersion string `json:"pricingVersion"`
RequestFingerprint string `json:"requestFingerprint"`
}
doJSON(t, server.URL, http.MethodPost, "/api/v1/pricing/estimate", apiKeyResponse.Secret, map[string]any{
"kind": "chat.completions", "model": pricingModel, "max_completion_tokens": 32,
"messages": []map[string]any{{"role": "user", "content": "estimate only"}},
}, http.StatusOK, &pricingEstimate)
if pricingEstimate.TotalAmount <= 0 || pricingEstimate.ReservationAmount < pricingEstimate.TotalAmount ||
pricingEstimate.CandidateCount != 1 || pricingEstimate.PricingVersion != "effective-pricing-v2" || pricingEstimate.RequestFingerprint == "" {
t.Fatalf("unexpected effective pricing estimate: %+v", pricingEstimate)
}
var estimateSideEffectsAfter struct {
Tasks int
Transactions int
Frozen string
}
if err := testPool.QueryRow(ctx, `
SELECT (SELECT count(*) FROM gateway_tasks),
(SELECT count(*) FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid),
(SELECT frozen_balance::text FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid AND currency='resource')`,
smokeGatewayUserID,
).Scan(&estimateSideEffectsAfter.Tasks, &estimateSideEffectsAfter.Transactions, &estimateSideEffectsAfter.Frozen); err != nil {
t.Fatalf("read estimate side effects after request: %v", err)
}
if estimateSideEffectsAfter != estimateSideEffectsBefore {
t.Fatalf("pricing estimate must be read-only, before=%+v after=%+v", estimateSideEffectsBefore, estimateSideEffectsAfter)
}
idempotencyPayload := map[string]any{
"model": pricingModel, "runMode": "simulation", "simulation": true, "simulationDurationMs": 5,
"messages": []map[string]any{{"role": "user", "content": "idempotent replay"}},
}
idempotencyHeaders := map[string]string{"Idempotency-Key": "chat-replay-" + suffixText}
var idempotentFirst map[string]any
firstReplayHeaders := doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusOK, &idempotentFirst)
var idempotentSecond map[string]any
secondReplayHeaders := doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusOK, &idempotentSecond)
idempotentTaskID := firstReplayHeaders.Get("X-Gateway-Task-Id")
if idempotentTaskID == "" || secondReplayHeaders.Get("X-Gateway-Task-Id") != idempotentTaskID || secondReplayHeaders.Get("Idempotent-Replayed") != "true" {
t.Fatalf("non-stream replay headers first=%v second=%v", firstReplayHeaders, secondReplayHeaders)
}
var idempotentAttempts int
if err := testPool.QueryRow(ctx, `SELECT count(*) FROM gateway_task_attempts WHERE task_id=$1::uuid`, idempotentTaskID).Scan(&idempotentAttempts); err != nil {
t.Fatalf("count idempotent task attempts: %v", err)
}
if idempotentAttempts != 1 {
t.Fatalf("idempotent request called upstream %d times, want 1", idempotentAttempts)
}
idempotencyPayload["messages"] = []map[string]any{{"role": "user", "content": "different request"}}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusConflict, nil)
streamPayload := map[string]any{
"model": pricingModel, "runMode": "simulation", "simulation": true, "simulationDurationMs": 5, "stream": true,
"messages": []map[string]any{{"role": "user", "content": "stream replay"}},
}
streamRaw, err := json.Marshal(streamPayload)
if err != nil {
t.Fatal(err)
}
streamKey := "stream-replay-" + suffixText
streamRequest, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/chat/completions", bytes.NewReader(streamRaw))
if err != nil {
t.Fatal(err)
}
streamRequest.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret)
streamRequest.Header.Set("Content-Type", "application/json")
streamRequest.Header.Set("Idempotency-Key", streamKey)
streamResponse, err := http.DefaultClient.Do(streamRequest)
if err != nil {
t.Fatalf("execute first idempotent stream: %v", err)
}
_, _ = io.ReadAll(streamResponse.Body)
_ = streamResponse.Body.Close()
if streamResponse.StatusCode != http.StatusOK || streamResponse.Header.Get("X-Gateway-Task-Id") == "" {
t.Fatalf("first idempotent stream status=%d headers=%v", streamResponse.StatusCode, streamResponse.Header)
}
doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, streamPayload,
map[string]string{"Idempotency-Key": streamKey}, http.StatusConflict, nil)
var pricingTask struct { var pricingTask struct {
Task struct { Task struct {
ID string `json:"id"` ID string `json:"id"`
@@ -845,7 +952,7 @@ WHERE gateway_user_id = $1::uuid
} }
doAPIV1ChatCompletionAndLoadTask(t, ctx, testPool, server.URL, apiKeyResponse.Secret, map[string]any{ doAPIV1ChatCompletionAndLoadTask(t, ctx, testPool, server.URL, apiKeyResponse.Secret, map[string]any{
"model": pricingModel, "model": pricingModel,
"runMode": "simulation", "runMode": "production",
"simulation": true, "simulation": true,
"simulationDurationMs": 5, "simulationDurationMs": 5,
"messages": []map[string]any{{"role": "user", "content": "priced ping"}}, "messages": []map[string]any{{"role": "user", "content": "priced ping"}},
@@ -853,6 +960,20 @@ WHERE gateway_user_id = $1::uuid
if pricingTask.Task.Status != "succeeded" || !floatNear(pricingTask.Task.FinalChargeAmount, 0.028) { if pricingTask.Task.Status != "succeeded" || !floatNear(pricingTask.Task.FinalChargeAmount, 0.028) {
t.Fatalf("custom pricing rule set should drive text billing, got task=%+v", pricingTask.Task) t.Fatalf("custom pricing rule set should drive text billing, got task=%+v", pricingTask.Task)
} }
settlementDeadline := time.Now().Add(3 * time.Second)
for {
var billingStatus string
if err := testPool.QueryRow(ctx, `SELECT billing_status FROM gateway_tasks WHERE id = $1::uuid`, pricingTask.Task.ID).Scan(&billingStatus); err != nil {
t.Fatalf("read pricing task billing status: %v", err)
}
if billingStatus == "settled" {
break
}
if time.Now().After(settlementDeadline) {
t.Fatalf("pricing task billing did not settle before deadline, status=%s", billingStatus)
}
time.Sleep(25 * time.Millisecond)
}
var walletBalanceAfter float64 var walletBalanceAfter float64
var walletSpentAfter float64 var walletSpentAfter float64
if err := testPool.QueryRow(ctx, ` if err := testPool.QueryRow(ctx, `
@@ -1484,7 +1605,7 @@ WHERE m.platform_id = $1::uuid
t.Fatalf("workspace task list should include persisted task records, got %+v", workspaceTaskList.Items) t.Fatalf("workspace task list should include persisted task records, got %+v", workspaceTaskList.Items)
} }
req, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/tasks/"+taskResponse.Task.ID+"/events", nil) req, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/tasks/"+pricingTask.Task.ID+"/events", nil)
if err != nil { if err != nil {
t.Fatalf("build events request: %v", err) t.Fatalf("build events request: %v", err)
} }
@@ -1564,6 +1685,7 @@ WHERE m.platform_id = $1::uuid
DatabaseURL: databaseURL, DatabaseURL: databaseURL,
IdentityMode: "hybrid", IdentityMode: "hybrid",
JWTSecret: "test-secret", JWTSecret: "test-secret",
BillingEngineMode: "enforce",
TaskProgressCallbackEnabled: true, TaskProgressCallbackEnabled: true,
TaskProgressCallbackURL: "http://callback.local/task-progress", TaskProgressCallbackURL: "http://callback.local/task-progress",
CORSAllowedOrigin: "*", CORSAllowedOrigin: "*",
@@ -1864,7 +1986,7 @@ func doJSON(t *testing.T, baseURL string, method string, path string, token stri
doJSONWithHeaders(t, baseURL, method, path, token, payload, nil, expectedStatus, out) doJSONWithHeaders(t, baseURL, method, path, token, payload, nil, expectedStatus, out)
} }
func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string, token string, payload any, headers map[string]string, expectedStatus int, out any) { func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string, token string, payload any, headers map[string]string, expectedStatus int, out any) http.Header {
t.Helper() t.Helper()
var body io.Reader var body io.Reader
if payload != nil { if payload != nil {
@@ -1901,16 +2023,22 @@ func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string,
t.Fatalf("decode %s %s response: %v body=%s", method, path, err, string(raw)) t.Fatalf("decode %s %s response: %v body=%s", method, path, err, string(raw))
} }
} }
return resp.Header.Clone()
} }
func doAPIV1ChatCompletionAndLoadTask(t *testing.T, ctx context.Context, pool *pgxpool.Pool, baseURL string, token string, payload map[string]any, marker string, expectedStatus int, responseOut any, taskDetailOut any) string { func doAPIV1ChatCompletionAndLoadTask(t *testing.T, ctx context.Context, pool *pgxpool.Pool, baseURL string, token string, payload map[string]any, marker string, expectedStatus int, responseOut any, taskDetailOut any) string {
t.Helper() t.Helper()
payload["integrationTestMarker"] = marker _ = ctx
_ = pool
_ = marker
if responseOut == nil { if responseOut == nil {
responseOut = &map[string]any{} responseOut = &map[string]any{}
} }
doJSON(t, baseURL, http.MethodPost, "/api/v1/chat/completions", token, payload, expectedStatus, responseOut) responseHeaders := doJSONWithHeaders(t, baseURL, http.MethodPost, "/api/v1/chat/completions", token, payload, nil, expectedStatus, responseOut)
taskID := waitForTaskIDByRequestField(t, ctx, pool, "integrationTestMarker", marker, 2*time.Second) taskID := strings.TrimSpace(responseHeaders.Get("X-Gateway-Task-Id"))
if taskID == "" {
t.Fatal("chat completion response did not expose X-Gateway-Task-Id")
}
if taskDetailOut != nil { if taskDetailOut != nil {
doJSON(t, baseURL, http.MethodGet, "/api/v1/tasks/"+taskID, token, nil, http.StatusOK, taskDetailOut) doJSON(t, baseURL, http.MethodGet, "/api/v1/tasks/"+taskID, token, nil, http.StatusOK, taskDetailOut)
} }
@@ -2046,11 +2174,6 @@ func waitForTaskStatus(t *testing.T, baseURL string, token string, taskID string
return detail return detail
} }
func waitForTaskIDByRequestMarker(t *testing.T, ctx context.Context, pool *pgxpool.Pool, marker string, timeout time.Duration) string {
t.Helper()
return waitForTaskIDByRequestField(t, ctx, pool, "cancelTestId", marker, timeout)
}
func waitForTaskIDByRequestField(t *testing.T, ctx context.Context, pool *pgxpool.Pool, key string, value string, timeout time.Duration) string { func waitForTaskIDByRequestField(t *testing.T, ctx context.Context, pool *pgxpool.Pool, key string, value string, timeout time.Duration) string {
t.Helper() t.Helper()
deadline := time.Now().Add(timeout) deadline := time.Now().Add(timeout)
apps/api/internal/httpapi/gemini_compat.go
+33 -4
View File
@@ -4,6 +4,7 @@ import (
"crypto/rand" "crypto/rand"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"mime" "mime"
@@ -94,6 +95,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusForbidden, "api key scope does not allow this capability") writeError(w, http.StatusForbidden, "api key scope does not allow this capability")
return return
} }
idempotencyKey, hasIdempotencyKey, err := optionalTaskIdempotencyKey(r)
if err != nil {
writeError(w, http.StatusBadRequest, "Idempotency-Key must contain one non-empty value", "invalid_idempotency_key")
return
}
prepared, err := s.prepareTaskRequest(r.Context(), r, user, mapping.Body) prepared, err := s.prepareTaskRequest(r.Context(), r, user, mapping.Body)
if err != nil { if err != nil {
s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err) s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err)
@@ -104,7 +110,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
writeError(w, status, err.Error(), clients.ErrorCode(err)) writeError(w, status, err.Error(), clients.ErrorCode(err))
return return
} }
task, err := s.store.CreateTask(r.Context(), store.CreateTaskInput{ createInput := store.CreateTaskInput{
Kind: mapping.Kind, Kind: mapping.Kind,
Model: mapping.Model, Model: mapping.Model,
RunMode: runModeFromRequest(prepared.Body), RunMode: runModeFromRequest(prepared.Body),
@@ -113,10 +119,33 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) {
ConversationID: prepared.ConversationID, ConversationID: prepared.ConversationID,
NewMessageCount: prepared.NewMessageCount, NewMessageCount: prepared.NewMessageCount,
MessageRefs: prepared.MessageRefs, MessageRefs: prepared.MessageRefs,
}, user) }
if hasIdempotencyKey {
createInput.IdempotencyKeyHash = taskIdempotencyKeyHash(idempotencyKey)
createInput.IdempotencyRequestHash = taskIdempotencyRequestHash(mapping.Kind, false, false, prepared.Body)
}
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
if err != nil { if err != nil {
s.logger.Error("create gemini task failed", "kind", mapping.Kind, "error", err) if errors.Is(err, store.ErrIdempotencyKeyReused) {
writeError(w, http.StatusInternalServerError, "create task failed") writeError(w, http.StatusConflict, "Idempotency-Key was reused for a different request", "idempotency_key_reused")
return
}
s.logger.Error("create gemini task failed", "kind", mapping.Kind, "error_category", "task_create_failed")
writeError(w, http.StatusInternalServerError, "create task failed", "task_create_failed")
return
}
task := created.Task
if created.Replayed {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent("idempotent_replay")
}
w.Header().Set("Idempotent-Replayed", "true")
w.Header().Set("X-Gateway-Task-Id", task.ID)
if task.Status == "succeeded" {
writeJSON(w, http.StatusOK, geminiGenerateContentResponse(task.Result, mapping.Model))
return
}
writeIdempotentTaskReplay(w, task, true)
return return
} }
runCtx, cancelRun := s.requestExecutionContext(r) runCtx, cancelRun := s.requestExecutionContext(r)
apps/api/internal/httpapi/handlers.go
+57 -4
View File
@@ -892,6 +892,7 @@ func (s *Server) deleteAPIKey(w http.ResponseWriter, r *http.Request) {
// @Failure 403 {object} ErrorEnvelope // @Failure 403 {object} ErrorEnvelope
// @Failure 404 {object} ErrorEnvelope // @Failure 404 {object} ErrorEnvelope
// @Failure 429 {object} ErrorEnvelope // @Failure 429 {object} ErrorEnvelope
// @Failure 503 {object} ErrorEnvelope
// @Failure 500 {object} ErrorEnvelope // @Failure 500 {object} ErrorEnvelope
// @Router /api/v1/pricing/estimate [post] // @Router /api/v1/pricing/estimate [post]
func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) { func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) {
@@ -916,10 +917,24 @@ func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) {
} }
estimate, err := s.runner.Estimate(r.Context(), kind, model, body, user) estimate, err := s.runner.Estimate(r.Context(), kind, model, body, user)
if err != nil { if err != nil {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent("estimate_failed")
}
if runner.IsPricingUnavailable(err) {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent("pricing_unavailable")
}
writeErrorWithDetails(w, http.StatusServiceUnavailable, runErrorMessage(err), runErrorDetails(err), "pricing_unavailable")
return
}
if errors.Is(err, store.ErrNoModelCandidate) { if errors.Is(err, store.ErrNoModelCandidate) {
writeErrorWithDetails(w, statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), store.ModelCandidateErrorCode(err)) writeErrorWithDetails(w, statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), store.ModelCandidateErrorCode(err))
return return
} }
if code := clients.ErrorCode(err); code == "bad_request" || code == "invalid_parameter" {
writeErrorWithDetails(w, http.StatusBadRequest, runErrorMessage(err), runErrorDetails(err), code)
return
}
s.logger.Error("estimate pricing failed", "error", err) s.logger.Error("estimate pricing failed", "error", err)
writeError(w, http.StatusInternalServerError, "estimate pricing failed") writeError(w, http.StatusInternalServerError, "estimate pricing failed")
return return
@@ -977,6 +992,7 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque
// @Produce json // @Produce json
// @Security BearerAuth // @Security BearerAuth
// @Param X-Async header bool false "true 时异步创建任务并返回 202" // @Param X-Async header bool false "true 时异步创建任务并返回 202"
// @Param Idempotency-Key header string false "可选请求幂等键；同一用户范围内唯一"
// @Param input body TaskRequest true "AI 任务请求，字段随任务类型变化" // @Param input body TaskRequest true "AI 任务请求，字段随任务类型变化"
// @Success 200 {object} CompatibleResponse // @Success 200 {object} CompatibleResponse
// @Success 202 {object} TaskAcceptedResponse // @Success 202 {object} TaskAcceptedResponse
@@ -1045,6 +1061,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
return return
} }
responsePlan := planTaskResponse(kind, compatible, body, r) responsePlan := planTaskResponse(kind, compatible, body, r)
idempotencyKey, hasIdempotencyKey, err := optionalTaskIdempotencyKey(r)
if err != nil {
writeError(w, http.StatusBadRequest, "Idempotency-Key must contain one non-empty value", "invalid_idempotency_key")
return
}
prepared, err := s.prepareTaskRequest(r.Context(), r, user, body) prepared, err := s.prepareTaskRequest(r.Context(), r, user, body)
if err != nil { if err != nil {
s.logger.Warn("prepare task request failed", "kind", kind, "error", err) s.logger.Warn("prepare task request failed", "kind", kind, "error", err)
@@ -1056,7 +1077,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
return return
} }
task, err := s.store.CreateTask(r.Context(), store.CreateTaskInput{ createInput := store.CreateTaskInput{
Kind: kind, Kind: kind,
Model: model, Model: model,
RunMode: runModeFromRequest(prepared.Body), RunMode: runModeFromRequest(prepared.Body),
@@ -1065,10 +1086,32 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler {
ConversationID: prepared.ConversationID, ConversationID: prepared.ConversationID,
NewMessageCount: prepared.NewMessageCount, NewMessageCount: prepared.NewMessageCount,
MessageRefs: prepared.MessageRefs, MessageRefs: prepared.MessageRefs,
}, user) }
if hasIdempotencyKey {
createInput.IdempotencyKeyHash = taskIdempotencyKeyHash(idempotencyKey)
createInput.IdempotencyRequestHash = taskIdempotencyRequestHash(kind, responsePlan.asyncMode, responsePlan.streamMode, prepared.Body)
}
created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user)
if err != nil { if err != nil {
s.logger.Error("create task failed", "kind", kind, "error", err) if errors.Is(err, store.ErrIdempotencyKeyReused) {
writeError(w, http.StatusInternalServerError, "create task failed") writeError(w, http.StatusConflict, "Idempotency-Key was reused for a different request", "idempotency_key_reused")
return
}
s.logger.Error("create task failed", "kind", kind, "error_category", "task_create_failed")
writeError(w, http.StatusInternalServerError, "create task failed", "task_create_failed")
return
}
task := created.Task
if created.Replayed {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent("idempotent_replay")
}
w.Header().Set("Idempotent-Replayed", "true")
if responsePlan.streamMode {
writeError(w, http.StatusConflict, "streaming idempotent replay is not supported", "idempotency_stream_replay_unsupported")
return
}
writeIdempotentTaskReplay(w, task, responsePlan.compatibleMode)
return return
} }
if responsePlan.asyncMode { if responsePlan.asyncMode {
@@ -1383,6 +1426,10 @@ func scopeForTaskKind(kind string) string {
func statusFromRunError(err error) int { func statusFromRunError(err error) int {
switch { switch {
case clients.ErrorCode(err) == "billing_hold":
return http.StatusServiceUnavailable
case runner.IsPricingUnavailable(err):
return http.StatusServiceUnavailable
case clients.ErrorCode(err) == "invalid_previous_response_id" || clients.ErrorCode(err) == "response_chain_too_deep" || clients.ErrorCode(err) == "unsupported_model_protocol" || clients.ErrorCode(err) == "unsupported_response_tool" || clients.ErrorCode(err) == "unsupported_response_parameter": case clients.ErrorCode(err) == "invalid_previous_response_id" || clients.ErrorCode(err) == "response_chain_too_deep" || clients.ErrorCode(err) == "unsupported_model_protocol" || clients.ErrorCode(err) == "unsupported_response_tool" || clients.ErrorCode(err) == "unsupported_response_parameter":
return http.StatusBadRequest return http.StatusBadRequest
case clients.ErrorCode(err) == "response_chain_unavailable": case clients.ErrorCode(err) == "response_chain_unavailable":
@@ -1411,6 +1458,9 @@ func statusFromRunError(err error) int {
} }
func runErrorCode(err error) string { func runErrorCode(err error) string {
if runner.IsPricingUnavailable(err) {
return "pricing_unavailable"
}
if errors.Is(err, store.ErrNoModelCandidate) { if errors.Is(err, store.ErrNoModelCandidate) {
return store.ModelCandidateErrorCode(err) return store.ModelCandidateErrorCode(err)
} }
@@ -1428,6 +1478,9 @@ func runErrorMessage(err error) string {
} }
func runErrorDetails(err error) map[string]any { func runErrorDetails(err error) map[string]any {
if detail := runner.PricingUnavailableDetails(err); len(detail) > 0 {
return map[string]any{"pricing": detail}
}
if detail := rateLimitErrorDetail(err); len(detail) > 0 { if detail := rateLimitErrorDetail(err); len(detail) > 0 {
return map[string]any{"rateLimit": detail} return map[string]any{"rateLimit": detail}
} }
apps/api/internal/httpapi/oidc_jit_integration_test.go
+17 -3
View File
@@ -43,7 +43,7 @@ func TestOIDCJITFullGatewayUserFlowAndSecurityBoundary(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("connect store: %v", err) t.Fatalf("connect store: %v", err)
} }
defer db.Close() t.Cleanup(db.Close)
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil { if err != nil {
@@ -247,6 +247,9 @@ DELETE FROM gateway_users WHERE source = 'oidc' AND external_user_id = ANY($1::t
t.Fatalf("rejected OIDC tokens created %d local users", rejectedWrites) t.Fatalf("rejected OIDC tokens created %d local users", rejectedWrites)
} }
if _, err := db.DisableActiveIdentityRevision(ctx, activeRevision.Version, "oidc-jit-disabled", "oidc-jit-disabled"); err != nil {
t.Fatalf("disable initial OIDC JIT revision: %v", err)
}
disabledJITRevision := prepareOIDCJITRevision(t, ctx, db, baseConfig, issuer, "default", false) disabledJITRevision := prepareOIDCJITRevision(t, ctx, db, baseConfig, issuer, "default", false)
testRevisionIDs = append(testRevisionIDs, disabledJITRevision.ID) testRevisionIDs = append(testRevisionIDs, disabledJITRevision.ID)
disabledJITRevision, _, err = db.ActivateIdentityRevision(ctx, disabledJITRevision.ID, disabledJITRevision.Version, "oidc-jit-disabled", "oidc-jit-disabled") disabledJITRevision, _, err = db.ActivateIdentityRevision(ctx, disabledJITRevision.ID, disabledJITRevision.Version, "oidc-jit-disabled", "oidc-jit-disabled")
@@ -298,6 +301,9 @@ func prepareOIDCJITRevision(t *testing.T, ctx context.Context, db *store.Store,
if err := secrets.Put(ctx, sessionReference, bytes.Repeat([]byte{8}, 32)); err != nil { if err := secrets.Put(ctx, sessionReference, bytes.Repeat([]byte{8}, 32)); err != nil {
t.Fatalf("store OIDC JIT session key: %v", err) t.Fatalf("store OIDC JIT session key: %v", err)
} }
if err := db.QueueIdentitySecretCleanup(ctx, sessionReference, time.Now().Add(10*time.Minute)); err != nil {
t.Fatalf("stage OIDC JIT session key for adoption: %v", err)
}
draft, err = db.ApplyIdentityManifest(ctx, draft.ID, draft.Version, identity.ManifestApplication{ draft, err = db.ApplyIdentityManifest(ctx, draft.ID, draft.Version, identity.ManifestApplication{
Manifest: identity.ManifestV1{ Manifest: identity.ManifestV1{
SchemaVersion: 1, Issuer: issuer, TenantID: oidcJITTenantID, ApplicationID: uuid.NewString(), SchemaVersion: 1, Issuer: issuer, TenantID: oidcJITTenantID, ApplicationID: uuid.NewString(),
@@ -360,6 +366,10 @@ var currentOIDCTestNonce string
func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie { func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie {
t.Helper() t.Helper()
parsedBaseURL, err := url.Parse(baseURL)
if err != nil {
t.Fatal(err)
}
jar, err := cookiejar.New(nil) jar, err := cookiejar.New(nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -371,6 +381,11 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie {
if len(via) > 10 { if len(via) > 10 {
return errors.New("too many redirects") return errors.New("too many redirects")
} }
// Follow the synthetic issuer's authorization redirect, then stop before
// the callback redirects the browser to the separately hosted web app.
if request.URL.Host != parsedBaseURL.Host && request.URL.Path != "/authorize" {
return http.ErrUseLastResponse
}
return nil return nil
}} }}
response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?returnTo=%2Fapi%2Fv1%2Fme") response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?returnTo=%2Fapi%2Fv1%2Fme")
@@ -378,11 +393,10 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie {
t.Fatalf("complete OIDC BFF login: %v", err) t.Fatalf("complete OIDC BFF login: %v", err)
} }
defer response.Body.Close() defer response.Body.Close()
if response.StatusCode != http.StatusOK { if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusSeeOther {
body, _ := io.ReadAll(response.Body) body, _ := io.ReadAll(response.Body)
t.Fatalf("OIDC BFF login status = %d, want 200: %s", response.StatusCode, body) t.Fatalf("OIDC BFF login status = %d, want 200: %s", response.StatusCode, body)
} }
parsedBaseURL, _ := url.Parse(baseURL)
for _, cookie := range jar.Cookies(parsedBaseURL) { for _, cookie := range jar.Cookies(parsedBaseURL) {
if cookie.Name == auth.OIDCSessionCookieName { if cookie.Name == auth.OIDCSessionCookieName {
if strings.Count(cookie.Value, ".") == 2 { if strings.Count(cookie.Value, ".") == 2 {
apps/api/internal/httpapi/openapi_models.go
+20 -2
View File
@@ -114,6 +114,18 @@ type AuditLogListResponse struct {
Items []store.AuditLog `json:"items"` Items []store.AuditLog `json:"items"`
} }
type BillingSettlementListResponse struct {
Items []store.BillingSettlement `json:"items"`
Total int `json:"total" example:"42"`
Page int `json:"page" example:"1"`
PageSize int `json:"pageSize" example:"50"`
}
type BillingSettlementRetryResponse struct {
Settlement store.BillingSettlement `json:"settlement"`
AuditLog store.AuditLog `json:"auditLog"`
}
type WalletTransactionListResponse struct { type WalletTransactionListResponse struct {
Items []store.GatewayWalletTransaction `json:"items"` Items []store.GatewayWalletTransaction `json:"items"`
Total int `json:"total" example:"42"` Total int `json:"total" example:"42"`
@@ -188,8 +200,14 @@ type PricingEstimateRequest struct {
} }
type PricingEstimateResponse struct { type PricingEstimateResponse struct {
Items []map[string]interface{} `json:"items"` Items []map[string]interface{} `json:"items"`
Resolver string `json:"resolver" example:"effective-pricing-v1"` Resolver string `json:"resolver" example:"effective-pricing-v2"`
TotalAmount float64 `json:"totalAmount" example:"1.25"`
ReservationAmount float64 `json:"reservationAmount" example:"2.75"`
Currency string `json:"currency" example:"resource"`
CandidateCount int `json:"candidateCount" example:"2"`
PricingVersion string `json:"pricingVersion" example:"effective-pricing-v2"`
RequestFingerprint string `json:"requestFingerprint" example:"76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f"`
} }
type TaskRequest struct { type TaskRequest struct {
apps/api/internal/httpapi/pricing_error_test.go
+27
View File
@@ -0,0 +1,27 @@
package httpapi
import (
"net/http"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/runner"
)
func TestPricingUnavailableUsesStructuredServiceUnavailableError(t *testing.T) {
err := &runner.PricingUnavailableError{
Reason: "missing, invalid, or not explicitly free",
ResourceType: "image",
RuleSetID: "rule-set-id",
}
if got := statusFromRunError(err); got != http.StatusServiceUnavailable {
t.Fatalf("statusFromRunError()=%d, want %d", got, http.StatusServiceUnavailable)
}
if got := runErrorCode(err); got != "pricing_unavailable" {
t.Fatalf("runErrorCode()=%q, want pricing_unavailable", got)
}
details := runErrorDetails(err)
pricing, _ := details["pricing"].(map[string]any)
if pricing["resourceType"] != "image" || pricing["ruleSetId"] != "rule-set-id" {
t.Fatalf("unexpected pricing details: %+v", details)
}
}
apps/api/internal/httpapi/pricing_handlers.go
+36
View File
@@ -5,6 +5,7 @@ import (
"errors" "errors"
"net/http" "net/http"
"strings" "strings"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
) )
@@ -146,10 +147,45 @@ func validPricingRuleSetInput(input store.PricingRuleSetInput) bool {
if strings.TrimSpace(input.RuleSetKey) == "" || strings.TrimSpace(input.Name) == "" || len(input.Rules) == 0 { if strings.TrimSpace(input.RuleSetKey) == "" || strings.TrimSpace(input.Name) == "" || len(input.Rules) == 0 {
return false return false
} }
if currency := strings.TrimSpace(input.Currency); currency != "" && currency != "resource" {
return false
}
for _, rule := range input.Rules { for _, rule := range input.Rules {
if strings.TrimSpace(rule.ResourceType) == "" || strings.TrimSpace(rule.Unit) == "" { if strings.TrimSpace(rule.ResourceType) == "" || strings.TrimSpace(rule.Unit) == "" {
return false return false
} }
if rule.BasePrice < 0 || (rule.BasePrice == 0 && !rule.IsFree) {
return false
}
if currency := strings.TrimSpace(rule.Currency); currency != "" && currency != "resource" {
return false
}
switch calculator := strings.TrimSpace(rule.CalculatorType); calculator {
case "", "token_usage", "unit_weight", "duration_weight":
default:
return false
}
calculator := strings.TrimSpace(rule.CalculatorType)
if calculator == "" {
calculator = store.DefaultEffectivePricingCalculator(rule.ResourceType)
}
if store.ValidateEffectivePricingRuleShape(strings.TrimSpace(rule.ResourceType), store.NormalizeEffectivePricingRuleUnit(rule.ResourceType, rule.Unit), calculator) != nil {
return false
}
effectiveFrom, fromOK := pricingEffectiveTime(rule.EffectiveFrom)
effectiveTo, toOK := pricingEffectiveTime(rule.EffectiveTo)
if !fromOK || !toOK || (!effectiveFrom.IsZero() && !effectiveTo.IsZero() && !effectiveFrom.Before(effectiveTo)) {
return false
}
} }
return true return true
} }
func pricingEffectiveTime(value string) (time.Time, bool) {
value = strings.TrimSpace(value)
if value == "" {
return time.Time{}, true
}
parsed, err := time.Parse(time.RFC3339, value)
return parsed, err == nil
}
apps/api/internal/httpapi/pricing_handlers_test.go
+28
View File
@@ -0,0 +1,28 @@
package httpapi
import (
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestPricingRuleSetRejectsImplicitZeroAndUnsupportedCalculator(t *testing.T) {
base := store.PricingRuleSetInput{
RuleSetKey: "test", Name: "test",
Rules: []store.PricingRuleInput{{
ResourceType: "image", Unit: "image", BasePrice: 0,
Currency: "resource", CalculatorType: "unit_weight",
}},
}
if validPricingRuleSetInput(base) {
t.Fatal("implicit zero price must be rejected")
}
base.Rules[0].IsFree = true
if !validPricingRuleSetInput(base) {
t.Fatal("explicit free price should be accepted")
}
base.Rules[0].CalculatorType = "formula"
if validPricingRuleSetInput(base) {
t.Fatal("arbitrary formula calculator must be rejected")
}
}
apps/api/internal/httpapi/server.go
+7 -2
View File
@@ -43,6 +43,7 @@ type Server struct {
identityTestRevision identity.Revision identityTestRevision identity.Revision
identityTestCookieSecure bool identityTestCookieSecure bool
identityTestBrowserEnabled bool identityTestBrowserEnabled bool
billingMetrics *ssfreceiver.Metrics
} }
type oidcPublicClient interface { type oidcPublicClient interface {
@@ -66,18 +67,19 @@ func NewServer(cfg config.Config, db *store.Store, logger *slog.Logger) http.Han
} }
func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Store, logger *slog.Logger) http.Handler { func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Store, logger *slog.Logger) http.Handler {
securityEventMetrics := &ssfreceiver.Metrics{}
server := &Server{ server := &Server{
ctx: ctx, ctx: ctx,
cfg: cfg, cfg: cfg,
store: db, store: db,
oidcUserResolver: db, oidcUserResolver: db,
auth: auth.New(cfg.JWTSecret, cfg.ServerMainBaseURL, cfg.ServerMainInternalToken), auth: auth.New(cfg.JWTSecret, cfg.ServerMainBaseURL, cfg.ServerMainInternalToken),
runner: runner.New(cfg, db, logger), runner: runner.New(cfg, db, logger, securityEventMetrics),
logger: logger, logger: logger,
billingMetrics: securityEventMetrics,
} }
server.auth.ServerMainInternalKey = cfg.ServerMainInternalKey server.auth.ServerMainInternalKey = cfg.ServerMainInternalKey
server.auth.ServerMainInternalSecret = cfg.ServerMainInternalSecret server.auth.ServerMainInternalSecret = cfg.ServerMainInternalSecret
securityEventMetrics := &ssfreceiver.Metrics{}
secretStore, err := identitySecretStore(cfg) secretStore, err := identitySecretStore(cfg)
if err != nil { if err != nil {
panic("invalid identity SecretStore: " + err.Error()) panic("invalid identity SecretStore: " + err.Error())
@@ -118,6 +120,7 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
} }
server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey
server.runner.StartAsyncQueueWorker(ctx) server.runner.StartAsyncQueueWorker(ctx)
server.runner.StartBillingSettlementWorker(ctx)
server.startLocalTempAssetCleanup(ctx) server.startLocalTempAssetCleanup(ctx)
server.startOIDCSessionCleanup(ctx) server.startOIDCSessionCleanup(ctx)
@@ -204,6 +207,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor
mux.Handle("DELETE /api/admin/runtime/policy-sets/{policySetID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteRuntimePolicySet))) mux.Handle("DELETE /api/admin/runtime/policy-sets/{policySetID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteRuntimePolicySet)))
mux.Handle("GET /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getRunnerPolicy))) mux.Handle("GET /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getRunnerPolicy)))
mux.Handle("PATCH /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRunnerPolicy))) mux.Handle("PATCH /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRunnerPolicy)))
mux.Handle("GET /api/admin/runtime/billing-settlements", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listBillingSettlements)))
mux.Handle("POST /api/admin/runtime/billing-settlements/{settlementId}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryBillingSettlement)))
mux.Handle("POST /api/admin/runtime/model-rate-limits/{platformModelID}/restore", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.restorePlatformModelRuntimeStatus))) mux.Handle("POST /api/admin/runtime/model-rate-limits/{platformModelID}/restore", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.restorePlatformModelRuntimeStatus)))
mux.Handle("GET /api/admin/config/network-proxy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getNetworkProxyConfig))) mux.Handle("GET /api/admin/config/network-proxy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getNetworkProxyConfig)))
mux.Handle("GET /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getFileStorageSettings))) mux.Handle("GET /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getFileStorageSettings)))
apps/api/internal/httpapi/task_idempotency.go
+85
View File
@@ -0,0 +1,85 @@
package httpapi
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"net/http"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
var errInvalidTaskIdempotencyKey = errors.New("invalid Idempotency-Key header")
func optionalTaskIdempotencyKey(r *http.Request) (string, bool, error) {
values := r.Header.Values("Idempotency-Key")
if len(values) == 0 {
return "", false, nil
}
if len(values) != 1 {
return "", false, errInvalidTaskIdempotencyKey
}
value := strings.TrimSpace(values[0])
if value == "" || len(value) > 255 || strings.Contains(value, ",") {
return "", false, errInvalidTaskIdempotencyKey
}
return value, true, nil
}
func taskIdempotencyKeyHash(key string) string {
digest := sha256.Sum256([]byte(key))
return hex.EncodeToString(digest[:])
}
func taskIdempotencyRequestHash(kind string, async bool, stream bool, body map[string]any) string {
payload, _ := json.Marshal(map[string]any{
"kind": kind, "async": async, "stream": stream, "request": body,
})
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:])
}
func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, compatible bool) {
w.Header().Set("Idempotent-Replayed", "true")
w.Header().Set("X-Gateway-Task-Id", task.ID)
if !compatible || task.AsyncMode || (task.Status != "succeeded" && task.Status != "failed" && task.Status != "cancelled") {
writeTaskAccepted(w, task)
return
}
if task.Status == "succeeded" {
writeJSON(w, http.StatusOK, task.Result)
return
}
status := storedTaskErrorStatus(task.ErrorCode)
message := strings.TrimSpace(task.ErrorMessage)
if message == "" {
message = strings.TrimSpace(task.Error)
}
if message == "" {
message = "task failed"
}
code := strings.TrimSpace(task.ErrorCode)
if code == "" {
code = "task_failed"
}
writeError(w, status, message, code)
}
func storedTaskErrorStatus(code string) int {
switch strings.TrimSpace(code) {
case "pricing_unavailable", "response_chain_unavailable", "billing_hold":
return http.StatusServiceUnavailable
case "insufficient_balance":
return http.StatusPaymentRequired
case "bad_request", "invalid_parameter", "invalid_previous_response_id", "unsupported_operation":
return http.StatusBadRequest
case "no_model_candidate", "cloned_voice_not_found":
return http.StatusNotFound
case "rate_limit", "platform_cooling_down", "model_cooling_down":
return http.StatusTooManyRequests
default:
return http.StatusBadGateway
}
}
apps/api/internal/httpapi/task_idempotency_test.go
+31
View File
@@ -0,0 +1,31 @@
package httpapi
import (
"net/http/httptest"
"testing"
)
func TestOptionalTaskIdempotencyKeyRejectsMultipleValues(t *testing.T) {
t.Parallel()
request := httptest.NewRequest("POST", "/", nil)
if _, present, err := optionalTaskIdempotencyKey(request); err != nil || present {
t.Fatalf("missing key present=%v err=%v", present, err)
}
request.Header.Add("Idempotency-Key", "one")
request.Header.Add("Idempotency-Key", "two")
if _, _, err := optionalTaskIdempotencyKey(request); err == nil {
t.Fatal("multiple keys must be rejected")
}
}
func TestTaskIdempotencyRequestHashIsCanonical(t *testing.T) {
t.Parallel()
first := map[string]any{"model": "m", "n": float64(1), "nested": map[string]any{"b": true, "a": "x"}}
second := map[string]any{"nested": map[string]any{"a": "x", "b": true}, "n": float64(1), "model": "m"}
if taskIdempotencyRequestHash("images.generations", false, false, first) != taskIdempotencyRequestHash("images.generations", false, false, second) {
t.Fatal("equivalent JSON objects must have the same request hash")
}
if taskIdempotencyRequestHash("images.generations", true, false, first) == taskIdempotencyRequestHash("images.generations", false, false, first) {
t.Fatal("async response semantics must be part of the request hash")
}
}
apps/api/internal/runner/billing_mode_test.go
+27
View File
@@ -0,0 +1,27 @@
package runner
import "testing"
func TestNormalizedBillingEngineMode(t *testing.T) {
t.Parallel()
if got := normalizedBillingEngineMode(""); got != "observe" {
t.Fatalf("empty mode = %q", got)
}
if got := normalizedBillingEngineMode("ENFORCE"); got != "enforce" {
t.Fatalf("enforce mode = %q", got)
}
if got := normalizedBillingEngineMode("hold"); got != "hold" {
t.Fatalf("hold mode = %q", got)
}
}
func TestBillingItemsFixedTotalKeepsNineDecimalPlaces(t *testing.T) {
t.Parallel()
items := []any{
map[string]any{"amount": "0.000000001"},
map[string]any{"amount": float64(0.000000002)},
}
if got := billingItemsFixedTotal(items).String(); got != "0.000000003" {
t.Fatalf("total = %s", got)
}
}
apps/api/internal/runner/billing_settlement_worker.go
+96
View File
@@ -0,0 +1,96 @@
package runner
import (
"context"
"errors"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
)
const billingSettlementPollInterval = time.Second
func billingSettlementRetryDelay(attempt int) time.Duration {
if attempt < 1 {
attempt = 1
}
delay := time.Second
for current := 1; current < attempt && delay < 15*time.Minute; current++ {
delay *= 2
}
if delay > 15*time.Minute {
return 15 * time.Minute
}
return delay
}
func billingSettlementErrorCode(err error) string {
if errors.Is(err, store.ErrInsufficientWalletBalance) {
return "insufficient_balance"
}
return "settlement_failed"
}
func billingSettlementErrorMessage(code string) string {
if code == "insufficient_balance" {
return "wallet balance is insufficient for settlement"
}
return "billing settlement processing failed"
}
func (s *Service) StartBillingSettlementWorker(ctx context.Context) {
workerID := "billing-" + uuid.NewString()
go s.runBillingSettlementWorker(ctx, workerID)
}
func (s *Service) runBillingSettlementWorker(ctx context.Context, workerID string) {
ticker := time.NewTicker(billingSettlementPollInterval)
defer ticker.Stop()
for {
s.processBillingSettlementBatch(ctx, workerID)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (s *Service) processBillingSettlementBatch(ctx context.Context, workerID string) {
items, err := s.store.ClaimBillingSettlements(ctx, workerID, store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout)
if err != nil {
if ctx.Err() == nil {
s.logger.Error("claim billing settlements failed", "error_category", "billing_settlement_claim_failed")
}
return
}
for _, item := range items {
if ctx.Err() != nil {
return
}
if err := s.store.ProcessBillingSettlement(ctx, item); err != nil {
code := billingSettlementErrorCode(err)
markErr := s.store.MarkBillingSettlementFailed(
context.WithoutCancel(ctx),
item,
code,
billingSettlementErrorMessage(code),
billingSettlementRetryDelay(item.Attempts),
)
if markErr != nil {
s.logger.Error("mark billing settlement failed", "settlementID", item.ID, "taskID", item.TaskID, "error_category", "billing_settlement_state_failed")
continue
}
s.observeBillingEvent("settlement_retry")
if item.Attempts >= store.BillingSettlementMaxAttempts {
s.observeBillingEvent("manual_review")
}
s.logger.Warn("billing settlement scheduled for retry", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action, "error_category", code, "attempt", item.Attempts)
continue
}
s.observeBillingEvent("settlement_completed")
s.logger.Debug("billing settlement completed", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action)
}
}
apps/api/internal/runner/billing_settlement_worker_test.go
+40
View File
@@ -0,0 +1,40 @@
package runner
import (
"errors"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestBillingSettlementRetryDelay(t *testing.T) {
t.Parallel()
tests := []struct {
attempt int
want time.Duration
}{
{attempt: 1, want: time.Second},
{attempt: 2, want: 2 * time.Second},
{attempt: 10, want: 512 * time.Second},
{attempt: 11, want: 15 * time.Minute},
{attempt: 20, want: 15 * time.Minute},
}
for _, test := range tests {
if got := billingSettlementRetryDelay(test.attempt); got != test.want {
t.Fatalf("attempt %d: got %s, want %s", test.attempt, got, test.want)
}
}
}
func TestBillingSettlementErrorCode(t *testing.T) {
t.Parallel()
if got := billingSettlementErrorCode(store.ErrInsufficientWalletBalance); got != "insufficient_balance" {
t.Fatalf("got %q", got)
}
if got := billingSettlementErrorCode(errors.New("boom")); got != "settlement_failed" {
t.Fatalf("got %q", got)
}
}
apps/api/internal/runner/execution_lease.go
+34
View File
@@ -0,0 +1,34 @@
package runner
import (
"context"
"errors"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
taskExecutionLeaseTTL = 5 * time.Minute
taskExecutionRenewInterval = 30 * time.Second
)
func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.CancelFunc, taskID string, executionToken string) {
ticker := time.NewTicker(taskExecutionRenewInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.store.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil {
if errors.Is(err, store.ErrTaskExecutionFinished) {
return
}
s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost")
cancel()
return
}
}
}
}
apps/api/internal/runner/pricing.go
+21 -13
View File
@@ -11,10 +11,14 @@ import (
) )
type EstimateResult struct { type EstimateResult struct {
Items []any `json:"items"` Items []any `json:"items"`
Resolver string `json:"resolver"` Resolver string `json:"resolver"`
TotalAmount float64 `json:"totalAmount"` TotalAmount float64 `json:"totalAmount"`
Currency string `json:"currency"` ReservationAmount float64 `json:"reservationAmount"`
Currency string `json:"currency"`
CandidateCount int `json:"candidateCount"`
PricingVersion string `json:"pricingVersion"`
RequestFingerprint string `json:"requestFingerprint"`
} }
func (s *Service) Estimate(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) (EstimateResult, error) { func (s *Service) Estimate(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) (EstimateResult, error) {
@@ -32,15 +36,19 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body
if err != nil { if err != nil {
return EstimateResult{}, err return EstimateResult{}, err
} }
candidate := candidates[0] estimates := make([]candidateEstimate, 0, len(candidates))
body = preprocessRequest(kind, body, candidate) for _, candidate := range candidates {
items := s.estimatedBillings(ctx, user, kind, body, candidate) preprocessing := s.preprocessRequestWithScripts(ctx, kind, body, candidate)
return EstimateResult{ if preprocessing.Err != nil {
Items: items, return EstimateResult{}, parameterPreprocessClientError(preprocessing.Err)
Resolver: "effective-pricing-v1", }
TotalAmount: totalBillingAmount(items), estimate, candidateErr := s.estimateCandidateV2(ctx, user, kind, preprocessing.Body, candidate)
Currency: billingCurrency(items), if candidateErr != nil {
}, nil return EstimateResult{}, candidateErr
}
estimates = append(estimates, estimate)
}
return buildEstimateResult(estimates, pricingRequestFingerprint(kind, model, body))
} }
func (s *Service) estimatedBillings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) []any { func (s *Service) estimatedBillings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) []any {
apps/api/internal/runner/pricing_v2.go
+759
View File
@@ -0,0 +1,759 @@
package runner
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math"
"math/big"
"strconv"
"strings"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/clients"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
const (
pricingVersionV2 = "effective-pricing-v2"
fixedScale = int64(1_000_000_000)
maxFixedAmount = fixedAmount(math.MaxInt64)
)
var ErrPricingUnavailable = errors.New("pricing unavailable")
var errFixedAmountOverflow = errors.New("fixed amount overflow")
type PricingUnavailableError struct {
Reason string
ResourceType string
RuleSetID string
}
func (e *PricingUnavailableError) Error() string {
message := "pricing unavailable"
if e.ResourceType != "" {
message += " for " + e.ResourceType
}
if e.Reason != "" {
message += ": " + e.Reason
}
return message
}
func (e *PricingUnavailableError) Unwrap() error { return ErrPricingUnavailable }
func isPricingUnavailable(err error) bool { return errors.Is(err, ErrPricingUnavailable) }
func IsPricingUnavailable(err error) bool { return errors.Is(err, ErrPricingUnavailable) }
func PricingUnavailableDetails(err error) map[string]any {
var pricingErr *PricingUnavailableError
if !errors.As(err, &pricingErr) {
return nil
}
details := map[string]any{"reason": pricingErr.Reason}
if pricingErr.ResourceType != "" {
details["resourceType"] = pricingErr.ResourceType
}
if pricingErr.RuleSetID != "" {
details["ruleSetId"] = pricingErr.RuleSetID
}
return details
}
// fixedAmount is a signed decimal with exactly nine fractional digits.
// Monetary decisions use this type; float64 is only produced at JSON-compatible boundaries.
type fixedAmount int64
func parseFixedAmount(value string) (fixedAmount, error) {
value = strings.TrimSpace(value)
if value == "" {
return 0, fmt.Errorf("empty decimal")
}
sign := int64(1)
if value[0] == '-' || value[0] == '+' {
if value[0] == '-' {
sign = -1
}
value = value[1:]
}
parts := strings.Split(value, ".")
if len(parts) > 2 || value == "" {
return 0, fmt.Errorf("invalid decimal %q", value)
}
whole := parts[0]
if whole == "" {
whole = "0"
}
fraction := ""
if len(parts) == 2 {
fraction = parts[1]
}
for _, digits := range []string{whole, fraction} {
for _, char := range digits {
if char < '0' || char > '9' {
return 0, fmt.Errorf("invalid decimal %q", value)
}
}
}
roundUp := false
if len(fraction) > 9 {
roundUp = fraction[9] >= '5'
fraction = fraction[:9]
}
fraction += strings.Repeat("0", 9-len(fraction))
combined := strings.TrimLeft(whole+fraction, "0")
if combined == "" {
combined = "0"
}
number := new(big.Int)
if _, ok := number.SetString(combined, 10); !ok {
return 0, fmt.Errorf("invalid decimal %q", value)
}
if roundUp {
number.Add(number, big.NewInt(1))
}
if !number.IsInt64() {
return 0, fmt.Errorf("decimal %q exceeds fixed amount range", value)
}
return fixedAmount(sign * number.Int64()), nil
}
func fixedAmountFromAny(value any) (fixedAmount, error) {
switch typed := value.(type) {
case fixedAmount:
return typed, nil
case string:
return parseFixedAmount(typed)
case json.Number:
return parseFixedAmount(typed.String())
case int:
return parseFixedAmount(strconv.FormatInt(int64(typed), 10))
case int64:
return parseFixedAmount(strconv.FormatInt(typed, 10))
case int32:
return parseFixedAmount(strconv.FormatInt(int64(typed), 10))
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) {
return 0, fmt.Errorf("non-finite decimal")
}
return parseFixedAmount(strconv.FormatFloat(typed, 'f', 9, 64))
case float32:
return fixedAmountFromAny(float64(typed))
default:
return 0, fmt.Errorf("unsupported decimal type %T", value)
}
}
func (a fixedAmount) String() string {
value := int64(a)
sign := ""
if value < 0 {
sign = "-"
value = -value
}
return fmt.Sprintf("%s%d.%09d", sign, value/fixedScale, value%fixedScale)
}
func (a fixedAmount) Float64() float64 { return float64(a) / float64(fixedScale) }
func (a fixedAmount) IsZero() bool { return a == 0 }
func addFixedAmounts(left fixedAmount, right fixedAmount) (fixedAmount, error) {
value := new(big.Int).Add(big.NewInt(int64(left)), big.NewInt(int64(right)))
return fixedAmountFromBigInt(value)
}
func multiplyFixedAmountByInt(amount fixedAmount, multiplier int) (fixedAmount, error) {
value := new(big.Int).Mul(big.NewInt(int64(amount)), big.NewInt(int64(multiplier)))
return fixedAmountFromBigInt(value)
}
func multiplyFixedAmounts(left fixedAmount, right fixedAmount) (fixedAmount, error) {
product := new(big.Int).Mul(big.NewInt(int64(left)), big.NewInt(int64(right)))
return fixedAmountFromBigInt(roundBigIntRatio(product, big.NewInt(fixedScale)))
}
func multiplyFixedAmountRatio(amount fixedAmount, numerator int, denominator int) (fixedAmount, error) {
if denominator == 0 {
return 0, fmt.Errorf("division by zero")
}
product := new(big.Int).Mul(big.NewInt(int64(amount)), big.NewInt(int64(numerator)))
return fixedAmountFromBigInt(roundBigIntRatio(product, big.NewInt(int64(denominator))))
}
func fixedAmountFromBigInt(value *big.Int) (fixedAmount, error) {
if value == nil || !value.IsInt64() {
return 0, errFixedAmountOverflow
}
return fixedAmount(value.Int64()), nil
}
func roundBigIntRatio(numerator *big.Int, denominator *big.Int) *big.Int {
negative := numerator.Sign() < 0
absolute := new(big.Int).Abs(new(big.Int).Set(numerator))
quotient, remainder := new(big.Int), new(big.Int)
quotient.QuoRem(absolute, denominator, remainder)
if new(big.Int).Mul(remainder, big.NewInt(2)).Cmp(denominator) >= 0 {
quotient.Add(quotient, big.NewInt(1))
}
if negative {
quotient.Neg(quotient)
}
return quotient
}
type resolvedPricing struct {
Config map[string]any
Currency string
RuleSetID string
RuleSetKey string
Source string
FreeResource map[string]bool
Snapshot map[string]any
}
func (pricing resolvedPricing) requiredPrice(resource string, keys ...string) (fixedAmount, error) {
if price, found, err := pricing.price(resource, keys...); err != nil {
return 0, &PricingUnavailableError{Reason: err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID}
} else if found && price > 0 {
return price, nil
} else if found && price == 0 && pricing.FreeResource[resource] {
return 0, nil
}
return 0, &PricingUnavailableError{Reason: "missing, invalid, or not explicitly free", ResourceType: resource, RuleSetID: pricing.RuleSetID}
}
func (pricing resolvedPricing) price(resource string, keys ...string) (fixedAmount, bool, error) {
for _, key := range keys {
if value, ok := pricing.Config[key]; ok {
amount, err := fixedAmountFromAny(value)
return amount, true, err
}
}
resourceConfig, _ := pricing.Config[resource].(map[string]any)
if len(resourceConfig) == 0 && resource == "image_edit" {
resourceConfig, _ = pricing.Config["image"].(map[string]any)
}
for _, key := range keys {
if value, ok := resourceConfig[key]; ok {
amount, err := fixedAmountFromAny(value)
return amount, true, err
}
}
if formula, ok := resourceConfig["formulaConfig"].(map[string]any); ok {
for _, key := range keys {
if value, exists := formula[key]; exists {
amount, err := fixedAmountFromAny(value)
return amount, true, err
}
}
}
if value, ok := resourceConfig["basePrice"]; ok {
amount, err := fixedAmountFromAny(value)
return amount, true, err
}
if resource == "text" {
return resolvedPricing{Config: pricing.Config}.price("text_total", keys...)
}
return 0, false, nil
}
func (pricing resolvedPricing) weight(resource string, key string, name string) (fixedAmount, error) {
if strings.TrimSpace(name) == "" {
return fixedAmount(fixedScale), nil
}
keys := weightKeyAliases(key)
names := weightValueAliases(key, name)
value, found := pricingWeightValue(pricing.Config, resource, keys, names)
if !found {
return fixedAmount(fixedScale), nil
}
weight, err := fixedAmountFromAny(value)
if err != nil || weight <= 0 {
reason := fmt.Sprintf("invalid %s weight for %s", key, name)
if err != nil {
reason += ": " + err.Error()
}
return 0, &PricingUnavailableError{Reason: reason, ResourceType: resource, RuleSetID: pricing.RuleSetID}
}
return weight, nil
}
func (pricing resolvedPricing) calculate(resource string, base fixedAmount, integerFactors []int, fixedFactors ...fixedAmount) (fixedAmount, error) {
amount := base
var err error
for _, factor := range integerFactors {
amount, err = multiplyFixedAmountByInt(amount, factor)
if err != nil {
return 0, pricing.calculationError(resource, err)
}
}
for _, factor := range fixedFactors {
amount, err = multiplyFixedAmounts(amount, factor)
if err != nil {
return 0, pricing.calculationError(resource, err)
}
}
return amount, nil
}
func (pricing resolvedPricing) calculateRatio(resource string, base fixedAmount, numerator int, denominator int, fixedFactors ...fixedAmount) (fixedAmount, error) {
amount, err := multiplyFixedAmountRatio(base, numerator, denominator)
if err != nil {
return 0, pricing.calculationError(resource, err)
}
return pricing.calculate(resource, amount, nil, fixedFactors...)
}
func (pricing resolvedPricing) add(resource string, left fixedAmount, right fixedAmount) (fixedAmount, error) {
amount, err := addFixedAmounts(left, right)
if err != nil {
return 0, pricing.calculationError(resource, err)
}
return amount, nil
}
func (pricing resolvedPricing) calculationError(resource string, err error) error {
return &PricingUnavailableError{Reason: "pricing calculation failed: " + err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID}
}
func pricingWeightValue(config map[string]any, resource string, keys []string, names []string) (any, bool) {
if value, ok := pricingWeightValueFromConfig(config, keys, names); ok {
return value, true
}
resourceConfig, _ := config[resource].(map[string]any)
if len(resourceConfig) == 0 && resource == "image_edit" {
resourceConfig, _ = config["image"].(map[string]any)
}
return pricingWeightValueFromConfig(resourceConfig, keys, names)
}
func pricingWeightValueFromConfig(config map[string]any, keys []string, names []string) (any, bool) {
if len(config) == 0 {
return nil, false
}
for _, key := range keys {
weights, _ := config[key].(map[string]any)
for _, name := range names {
if value, ok := weights[name]; ok {
return value, true
}
}
}
dynamic, _ := config["dynamicWeight"].(map[string]any)
for _, name := range names {
if value, ok := dynamic[name]; ok {
return value, true
}
}
for _, key := range keys {
weights, _ := dynamic[key].(map[string]any)
for _, name := range names {
if value, ok := weights[name]; ok {
return value, true
}
}
}
return nil, false
}
type candidateEstimate struct {
Items []any
Amount fixedAmount
Currency string
Snapshot map[string]any
Pricing resolvedPricing
}
func buildEstimateResult(estimates []candidateEstimate, fingerprint string) (EstimateResult, error) {
if len(estimates) == 0 {
return EstimateResult{}, &PricingUnavailableError{Reason: "no candidate has effective pricing"}
}
preferred := estimates[0]
reservation := preferred.Amount
currency := preferred.Currency
if currency == "" {
currency = "resource"
}
for _, estimate := range estimates[1:] {
candidateCurrency := estimate.Currency
if candidateCurrency == "" {
candidateCurrency = "resource"
}
if candidateCurrency != currency {
return EstimateResult{}, &PricingUnavailableError{Reason: "candidate currencies do not match"}
}
if estimate.Amount > reservation {
reservation = estimate.Amount
}
}
return EstimateResult{
Items: preferred.Items,
Resolver: pricingVersionV2,
TotalAmount: preferred.Amount.Float64(),
ReservationAmount: reservation.Float64(),
Currency: currency,
CandidateCount: len(estimates),
PricingVersion: pricingVersionV2,
RequestFingerprint: fingerprint,
}, nil
}
func estimatedOutputTokens(body map[string]any, candidate store.RuntimeModelCandidate) int {
for _, key := range []string{"max_completion_tokens", "max_output_tokens", "max_tokens"} {
if value, ok := positiveInteger(body[key]); ok {
return value
}
}
if limit, _, _, ok := candidateMaxOutputTokens(candidate, candidate.ModelType); ok {
return limit
}
return 4096
}
func pricingRequestFingerprint(kind string, model string, body map[string]any) string {
payload, _ := json.Marshal(map[string]any{"kind": kind, "model": model, "request": body})
digest := sha256.Sum256(payload)
return hex.EncodeToString(digest[:])
}
func (s *Service) resolveEffectivePricing(ctx context.Context, candidate store.RuntimeModelCandidate) (resolvedPricing, error) {
ruleSetID := firstNonEmptyString(
candidate.ModelPricingRuleSetID,
candidate.PlatformPricingRuleSetID,
candidate.BasePricingRuleSetID,
)
if ruleSetID != "" {
if s.store == nil {
return resolvedPricing{}, &PricingUnavailableError{Reason: "pricing store is unavailable", RuleSetID: ruleSetID}
}
config, err := s.store.PricingRuleSetBillingConfigV2(ctx, ruleSetID)
if err != nil {
return resolvedPricing{}, &PricingUnavailableError{Reason: err.Error(), RuleSetID: ruleSetID}
}
if len(candidate.BillingConfigOverride) > 0 {
config.Config = mergeMap(config.Config, candidate.BillingConfigOverride)
}
return resolvedPricing{
Config: config.Config, Currency: config.Currency, RuleSetID: config.RuleSetID,
RuleSetKey: config.RuleSetKey, Source: pricingRuleSource(candidate, ruleSetID),
FreeResource: config.FreeResource, Snapshot: config.Snapshot,
}, nil
}
config := candidate.BaseBillingConfig
source := "base_model_config"
if len(candidate.BillingConfig) > 0 {
config = candidate.BillingConfig
source = "platform_model_config"
}
if len(candidate.BillingConfigOverride) > 0 {
config = mergeMap(config, candidate.BillingConfigOverride)
source += "_override"
}
if len(config) == 0 {
return resolvedPricing{}, &PricingUnavailableError{Reason: "candidate has no pricing configuration"}
}
freeResource := explicitFreeResources(config)
snapshot := map[string]any{
"pricingVersion": pricingVersionV2,
"source": source,
"currency": "resource",
"config": config,
}
return resolvedPricing{
Config: config, Currency: "resource", Source: source,
FreeResource: freeResource, Snapshot: snapshot,
}, nil
}
func pricingRuleSource(candidate store.RuntimeModelCandidate, ruleSetID string) string {
switch ruleSetID {
case candidate.ModelPricingRuleSetID:
return "platform_model_rule_set"
case candidate.PlatformPricingRuleSetID:
return "platform_rule_set"
default:
return "base_model_rule_set"
}
}
func explicitFreeResources(config map[string]any) map[string]bool {
resources := map[string]bool{}
if isFree, _ := config["isFree"].(bool); isFree {
for _, resource := range []string{"text_input", "text_cached_input", "text_output", "text_total", "image", "image_edit", "video", "music", "audio"} {
resources[resource] = true
}
}
for _, resource := range []string{"text", "text_total", "image", "image_edit", "video", "music", "audio"} {
resourceConfig, _ := config[resource].(map[string]any)
if isFree, _ := resourceConfig["isFree"].(bool); isFree {
resources[resource] = true
}
}
return resources
}
func (s *Service) estimateCandidateV2(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) (candidateEstimate, error) {
usage := clients.Usage{InputTokens: estimateRequestTokens(body)}
if isTextGenerationKind(kind) {
usage.OutputTokens = estimatedOutputTokens(body, candidate)
}
usage.TotalTokens = usage.InputTokens + usage.OutputTokens
response := clients.Response{Usage: usage}
items, total, pricing, err := s.billingsV2(ctx, user, kind, body, candidate, response, true)
if err != nil {
return candidateEstimate{}, err
}
if isTextBillingKind(kind) {
if _, err := pricing.requiredTextPrice("text_cached_input", "textCachedInputPer1k", "cachedInputTokenPrice", "cacheInputTokenPrice", "textInputCacheHitPer1k", "inputCacheHitTokenPrice"); err != nil {
return candidateEstimate{}, err
}
}
return candidateEstimate{
Items: items, Amount: total, Currency: pricing.Currency, Snapshot: pricing.Snapshot, Pricing: pricing,
}, nil
}
func (s *Service) billingsV2(
ctx context.Context,
user *auth.User,
kind string,
body map[string]any,
candidate store.RuntimeModelCandidate,
response clients.Response,
simulated bool,
) ([]any, fixedAmount, resolvedPricing, error) {
pricing, err := s.resolveEffectivePricing(ctx, candidate)
if err != nil {
return nil, 0, resolvedPricing{}, err
}
return s.billingsWithResolvedPricingV2(ctx, user, kind, body, candidate, response, simulated, pricing)
}
func (s *Service) billingsWithResolvedPricingV2(
ctx context.Context,
user *auth.User,
kind string,
body map[string]any,
candidate store.RuntimeModelCandidate,
response clients.Response,
simulated bool,
pricing resolvedPricing,
) ([]any, fixedAmount, resolvedPricing, error) {
discount, err := fixedAmountFromAny(effectiveDiscount(ctx, s.store, user, candidate))
if err != nil || discount <= 0 {
return nil, 0, resolvedPricing{}, &PricingUnavailableError{Reason: "invalid discount factor", RuleSetID: pricing.RuleSetID}
}
buildLine := func(resourceType string, unit string, quantity any, amount fixedAmount, details map[string]any) map[string]any {
line := billingLineWithDetails(candidate, resourceType, unit, quantity, amount.Float64(), discount.Float64(), simulated, details)
line["pricingVersion"] = pricingVersionV2
line["pricingSource"] = pricing.Source
if pricing.RuleSetID != "" {
line["pricingRuleSetId"] = pricing.RuleSetID
}
line["isFree"] = amount.IsZero()
return line
}
if isTextBillingKind(kind) {
inputTokens := response.Usage.InputTokens
outputTokens := response.Usage.OutputTokens
cachedInputTokens := response.Usage.CachedInputTokens
if isTextInputOnlyKind(kind) && inputTokens == 0 && response.Usage.TotalTokens > 0 {
inputTokens = response.Usage.TotalTokens
}
if inputTokens == 0 && outputTokens == 0 {
inputTokens = estimateRequestTokens(body)
if isTextGenerationKind(kind) {
outputTokens = 1
}
}
if cachedInputTokens > inputTokens && inputTokens > 0 {
cachedInputTokens = inputTokens
}
uncachedInputTokens := inputTokens - cachedInputTokens
if uncachedInputTokens < 0 {
uncachedInputTokens = 0
}
inputPrice, err := pricing.requiredTextPrice("text_input", "textInputPer1k", "inputTokenPrice", "basePrice")
if err != nil {
return nil, 0, resolvedPricing{}, err
}
items := make([]any, 0, 3)
total := fixedAmount(0)
if uncachedInputTokens > 0 || cachedInputTokens == 0 {
amount, calculationErr := pricing.calculateRatio("text_input", inputPrice, uncachedInputTokens, 1000, discount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
total, calculationErr = pricing.add("text", total, amount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
items = append(items, buildLine("text_input", "1k_tokens", uncachedInputTokens, amount, map[string]any{
"inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens,
"cachedInputTokens": cachedInputTokens, "pricePer1k": inputPrice.Float64(),
}))
}
if cachedInputTokens > 0 {
cachedPrice, priceErr := pricing.requiredTextPrice("text_cached_input", "textCachedInputPer1k", "cachedInputTokenPrice", "cacheInputTokenPrice", "textInputCacheHitPer1k", "inputCacheHitTokenPrice")
if priceErr != nil {
return nil, 0, resolvedPricing{}, priceErr
}
amount, calculationErr := pricing.calculateRatio("text_cached_input", cachedPrice, cachedInputTokens, 1000, discount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
total, calculationErr = pricing.add("text", total, amount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
items = append(items, buildLine("text_cached_input", "1k_tokens", cachedInputTokens, amount, map[string]any{
"inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens,
"cachedInputTokens": cachedInputTokens, "pricePer1k": cachedPrice.Float64(),
}))
}
if isTextGenerationKind(kind) {
outputPrice, priceErr := pricing.requiredTextPrice("text_output", "textOutputPer1k", "outputTokenPrice", "basePrice")
if priceErr != nil {
return nil, 0, resolvedPricing{}, priceErr
}
amount, calculationErr := pricing.calculateRatio("text_output", outputPrice, outputTokens, 1000, discount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
total, calculationErr = pricing.add("text", total, amount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
items = append(items, buildLine("text_output", "1k_tokens", outputTokens, amount, map[string]any{"pricePer1k": outputPrice.Float64()}))
}
return items, total, pricing, nil
}
count := requestOutputCount(body)
resource := "image"
unit := "image"
baseKey := "imageBase"
if kind == "images.edits" {
resource = "image_edit"
baseKey = "editBase"
}
if kind == "videos.generations" {
resource = "video"
unit = "5s_video"
baseKey = "videoBase"
duration, durationSource := billingDurationSeconds(body, response)
audioEnabled, audioSource := billingAudioEnabled(body, response)
durationUnits := int(math.Max(1, math.Ceil(duration/5)))
price, priceErr := pricing.requiredPrice(resource, baseKey, "basePrice")
if priceErr != nil {
return nil, 0, resolvedPricing{}, priceErr
}
resolutionWeight, weightErr := pricing.weight(resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size")))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
audioWeight, weightErr := pricing.weight(resource, "audioWeights", boolWeightKey(audioEnabled))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
referenceVideoWeight, weightErr := pricing.weight(resource, "referenceVideoWeights", boolWeightKey(requestHasReferenceVideo(body)))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
voiceWeight, weightErr := pricing.weight(resource, "voiceSpecifiedWeights", boolWeightKey(requestHasVoiceID(body, audioEnabled)))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
amount, calculationErr := pricing.calculate(resource, price, []int{count, durationUnits}, resolutionWeight, audioWeight, referenceVideoWeight, voiceWeight, discount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
item := buildLine(resource, unit, count*durationUnits, amount, map[string]any{
"count": count, "audio": audioEnabled, "audioSource": audioSource,
"durationSeconds": duration, "durationSource": durationSource,
"durationUnit": "5s", "durationUnitCount": durationUnits,
})
return []any{item}, amount, pricing, nil
}
if kind == "song.generations" || kind == "music.generations" {
resource = "music"
unit = "song"
baseKey = "musicBase"
}
if kind == "speech.generations" || kind == "voice.clone" {
resource = "audio"
unit = "character"
baseKey = "audioBase"
count = len([]rune(stringFromMap(body, "text")))
if count <= 0 {
count = 1
}
}
price, err := pricing.requiredPrice(resource, baseKey, "basePrice")
if err != nil {
return nil, 0, resolvedPricing{}, err
}
amount, calculationErr := pricing.calculate(resource, price, []int{count})
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
if resource == "image" || resource == "image_edit" {
qualityWeight, weightErr := pricing.weight(resource, "qualityWeights", stringFromMap(body, "quality"))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
sizeWeight, weightErr := pricing.weight(resource, "sizeWeights", stringFromMap(body, "size"))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
resolutionWeight, weightErr := pricing.weight(resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size")))
if weightErr != nil {
return nil, 0, resolvedPricing{}, weightErr
}
amount, calculationErr = pricing.calculate(resource, amount, nil, qualityWeight, sizeWeight, resolutionWeight)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
}
amount, calculationErr = pricing.calculate(resource, amount, nil, discount)
if calculationErr != nil {
return nil, 0, resolvedPricing{}, calculationErr
}
return []any{buildLine(resource, unit, count, amount, nil)}, amount, pricing, nil
}
func (pricing resolvedPricing) requiredTextPrice(resource string, keys ...string) (fixedAmount, error) {
price, found, err := pricing.price("text", keys...)
if err != nil {
return 0, &PricingUnavailableError{Reason: err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID}
}
if found && price > 0 {
return price, nil
}
if found && price == 0 && (pricing.FreeResource[resource] || pricing.FreeResource["text_total"] || pricing.FreeResource["text"]) {
return 0, nil
}
return 0, &PricingUnavailableError{Reason: "missing, invalid, or not explicitly free", ResourceType: resource, RuleSetID: pricing.RuleSetID}
}
func maximumCandidateEstimate(estimates []candidateEstimate) (candidateEstimate, error) {
if len(estimates) == 0 {
return candidateEstimate{}, &PricingUnavailableError{Reason: "no candidate has effective pricing"}
}
maximum := estimates[0]
for _, estimate := range estimates[1:] {
if estimate.Currency != maximum.Currency {
return candidateEstimate{}, &PricingUnavailableError{Reason: "candidate currencies do not match"}
}
if estimate.Amount > maximum.Amount {
maximum = estimate
}
}
return maximum, nil
}
apps/api/internal/runner/pricing_v2_test.go
+148
View File
@@ -0,0 +1,148 @@
package runner
import (
"errors"
"math"
"testing"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
)
func TestFixedAmountPreservesNineDecimalPlaces(t *testing.T) {
amount, err := parseFixedAmount("0.123456789")
if err != nil {
t.Fatalf("parse fixed amount: %v", err)
}
if got, want := amount.String(), "0.123456789"; got != want {
t.Fatalf("amount.String()=%q, want %q", got, want)
}
price := mustFixedAmount(t, "0.000000001")
product, err := multiplyFixedAmountByInt(price, 3)
if err != nil {
t.Fatalf("multiply fixed amount: %v", err)
}
if got, want := product.String(), "0.000000003"; got != want {
t.Fatalf("nano amount multiplication=%q, want %q", got, want)
}
}
func TestFixedAmountOperationsRejectOverflow(t *testing.T) {
maximum := fixedAmount(math.MaxInt64)
if _, err := multiplyFixedAmountByInt(maximum, 2); !errors.Is(err, errFixedAmountOverflow) {
t.Fatalf("multiply overflow error=%v", err)
}
if _, err := addFixedAmounts(maximum, 1); !errors.Is(err, errFixedAmountOverflow) {
t.Fatalf("add overflow error=%v", err)
}
pricing := resolvedPricing{RuleSetID: "overflow-rule"}
if _, err := pricing.calculate("image", maximum, []int{2}); !isPricingUnavailable(err) {
t.Fatalf("pricing overflow should be unavailable: %v", err)
}
}
func TestEstimatedOutputTokensUsesAliasesAndCapabilityFallback(t *testing.T) {
candidate := store.RuntimeModelCandidate{
ModelType: "text_generate",
Capabilities: map[string]any{
"text_generate": map[string]any{"max_output_tokens": 8192},
},
}
tests := []struct {
name string
body map[string]any
want int
}{
{name: "max completion has highest priority", body: map[string]any{"max_completion_tokens": 700, "max_output_tokens": 600, "max_tokens": 500}, want: 700},
{name: "max output precedes legacy max tokens", body: map[string]any{"max_output_tokens": 600, "max_tokens": 500}, want: 600},
{name: "legacy max tokens", body: map[string]any{"max_tokens": 500}, want: 500},
{name: "model capability fallback", body: map[string]any{}, want: 8192},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := estimatedOutputTokens(test.body, candidate); got != test.want {
t.Fatalf("estimatedOutputTokens()=%d, want %d", got, test.want)
}
})
}
if got := estimatedOutputTokens(nil, store.RuntimeModelCandidate{}); got != 4096 {
t.Fatalf("missing capability fallback=%d, want 4096", got)
}
}
func TestBuildEstimateResultUsesPreferredTotalAndMaximumReservation(t *testing.T) {
result, err := buildEstimateResult([]candidateEstimate{
{Items: []any{map[string]any{"amount": 1.25, "currency": "resource"}}, Amount: mustFixedAmount(t, "1.25")},
{Items: []any{map[string]any{"amount": 2.75, "currency": "resource"}}, Amount: mustFixedAmount(t, "2.75")},
}, "fingerprint")
if err != nil {
t.Fatalf("build estimate result: %v", err)
}
if result.TotalAmount != 1.25 {
t.Fatalf("preferred total=%v, want 1.25", result.TotalAmount)
}
if result.ReservationAmount != 2.75 {
t.Fatalf("reservation=%v, want 2.75", result.ReservationAmount)
}
if result.CandidateCount != 2 || result.PricingVersion != pricingVersionV2 || result.RequestFingerprint != "fingerprint" {
t.Fatalf("unexpected estimate metadata: %+v", result)
}
}
func TestPricingAvailabilityRequiresExplicitFree(t *testing.T) {
paid := resolvedPricing{Config: map[string]any{"imageBase": 1.5}, Currency: "resource"}
if _, err := paid.requiredPrice("image", "imageBase", "basePrice"); err != nil {
t.Fatalf("positive price should be available: %v", err)
}
missing := resolvedPricing{Config: map[string]any{}, Currency: "resource"}
if _, err := missing.requiredPrice("image", "imageBase", "basePrice"); !isPricingUnavailable(err) {
t.Fatalf("missing price should be unavailable, got %v", err)
}
ordinaryZero := resolvedPricing{Config: map[string]any{"imageBase": 0}, Currency: "resource"}
if _, err := ordinaryZero.requiredPrice("image", "imageBase", "basePrice"); !isPricingUnavailable(err) {
t.Fatalf("ordinary zero should be unavailable, got %v", err)
}
explicitFree := resolvedPricing{
Config: map[string]any{"imageBase": 0},
Currency: "resource",
FreeResource: map[string]bool{"image": true},
}
price, err := explicitFree.requiredPrice("image", "imageBase", "basePrice")
if err != nil || !price.IsZero() {
t.Fatalf("explicit free should resolve to zero: price=%s err=%v", price.String(), err)
}
}
func TestPricingWeightsUseFixedPrecisionAndRejectInvalidValues(t *testing.T) {
pricing := resolvedPricing{Config: map[string]any{
"image": map[string]any{
"dynamicWeight": map[string]any{
"qualityFactors": map[string]any{"high": "1.123456789", "broken": 0},
},
},
}}
weight, err := pricing.weight("image", "qualityWeights", "high")
if err != nil || weight.String() != "1.123456789" {
t.Fatalf("exact weight=%s err=%v", weight.String(), err)
}
if _, err := pricing.weight("image", "qualityWeights", "broken"); !isPricingUnavailable(err) {
t.Fatalf("invalid configured weight should make pricing unavailable: %v", err)
}
defaultWeight, err := pricing.weight("image", "qualityWeights", "unconfigured")
if err != nil || defaultWeight.String() != "1.000000000" {
t.Fatalf("default weight=%s err=%v", defaultWeight.String(), err)
}
}
func mustFixedAmount(t *testing.T, value string) fixedAmount {
t.Helper()
amount, err := parseFixedAmount(value)
if err != nil {
t.Fatalf("parse %q: %v", value, err)
}
return amount
}
apps/api/internal/runner/queue_worker.go
+23 -3
View File
@@ -10,6 +10,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
"github.com/riverqueue/river" "github.com/riverqueue/river"
"github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/riverdriver/riverpgxv5"
"github.com/riverqueue/river/rivermigrate" "github.com/riverqueue/river/rivermigrate"
@@ -38,16 +39,26 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs
if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" { if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" {
return nil return nil
} }
result, runErr := w.service.Execute(ctx, task, authUserFromTask(task)) executionToken := uuid.NewString()
result, runErr := w.service.executeWithToken(ctx, task, authUserFromTask(task), nil, executionToken)
if runErr == nil { if runErr == nil {
w.service.logger.Debug("river async task completed", "taskID", task.ID, "status", result.Task.Status, "riverJobID", job.ID) w.service.logger.Debug("river async task completed", "taskID", task.ID, "status", result.Task.Status, "riverJobID", job.ID)
return nil return nil
} }
if errors.Is(runErr, store.ErrTaskExecutionLeaseUnavailable) {
w.service.logger.Debug("river async task execution lease already held", "taskID", task.ID, "riverJobID", job.ID)
return nil
}
if errors.Is(runErr, store.ErrTaskExecutionManualReview) {
w.service.logger.Warn("river async task moved to manual review after ambiguous upstream submission", "taskID", task.ID, "riverJobID", job.ID)
return nil
}
var queuedErr *TaskQueuedError var queuedErr *TaskQueuedError
if errors.As(runErr, &queuedErr) { if errors.As(runErr, &queuedErr) {
return river.JobSnooze(queuedErr.Delay) return river.JobSnooze(queuedErr.Delay)
} }
if ctx.Err() != nil { if ctx.Err() != nil {
task.ExecutionToken = executionToken
queued, queueErr := w.service.requeueInterruptedAsyncTask(context.WithoutCancel(ctx), task) queued, queueErr := w.service.requeueInterruptedAsyncTask(context.WithoutCancel(ctx), task)
if queueErr != nil { if queueErr != nil {
return queueErr return queueErr
@@ -145,8 +156,7 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error {
return err return err
} }
for _, item := range items { for _, item := range items {
task := store.GatewayTask{ID: item.ID} result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item))
result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskInsertOpts(task))
if err != nil { if err != nil {
return err return err
} }
@@ -186,6 +196,16 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts {
} }
} }
func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem) *river.InsertOpts {
opts := asyncTaskInsertOpts(store.GatewayTask{ID: item.ID})
opts.ScheduledAt = item.NextRunAt
// A replacement process must not be blocked by a River row that the dead
// process left in running state. PostgreSQL execution leases still ensure
// that only one recovery job can call the upstream provider.
opts.UniqueOpts = river.UniqueOpts{}
return opts
}
func authUserFromTask(task store.GatewayTask) *auth.User { func authUserFromTask(task store.GatewayTask) *auth.User {
roles := []string{"user"} roles := []string{"user"}
if strings.TrimSpace(task.UserID) == "" { if strings.TrimSpace(task.UserID) == "" {
apps/api/internal/runner/service.go
+340 -78
View File
@@ -14,6 +14,7 @@ import (
"github.com/easyai/easyai-ai-gateway/apps/api/internal/config" "github.com/easyai/easyai-ai-gateway/apps/api/internal/config"
scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script" scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/store" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store"
"github.com/google/uuid"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/riverqueue/river" "github.com/riverqueue/river"
) )
@@ -26,6 +27,11 @@ type Service struct {
scriptExecutor *scriptengine.Executor scriptExecutor *scriptengine.Executor
httpClients *httpClientCache httpClients *httpClientCache
riverClient *river.Client[pgx.Tx] riverClient *river.Client[pgx.Tx]
billingMetrics billingMetricsObserver
}
type billingMetricsObserver interface {
ObserveBillingEvent(string)
} }
type Result struct { type Result struct {
@@ -39,6 +45,19 @@ type TaskQueuedError struct {
Delay time.Duration Delay time.Duration
} }
type upstreamSubmissionUnknownError struct {
AttemptID string
Cause error
}
func (e *upstreamSubmissionUnknownError) Error() string {
return "upstream submission result is unknown"
}
func (e *upstreamSubmissionUnknownError) Unwrap() error {
return e.Cause
}
func (e *TaskQueuedError) Error() string { func (e *TaskQueuedError) Error() string {
return ErrTaskQueued.Error() return ErrTaskQueued.Error()
} }
@@ -47,10 +66,10 @@ func (e *TaskQueuedError) Is(target error) bool {
return target == ErrTaskQueued return target == ErrTaskQueued
} }
func New(cfg config.Config, db *store.Store, logger *slog.Logger) *Service { func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...billingMetricsObserver) *Service {
httpClients := newHTTPClientCache() httpClients := newHTTPClientCache()
scriptExecutor := &scriptengine.Executor{Logger: logger} scriptExecutor := &scriptengine.Executor{Logger: logger}
return &Service{ service := &Service{
cfg: cfg, cfg: cfg,
store: db, store: db,
logger: logger, logger: logger,
@@ -76,6 +95,16 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger) *Service {
}, },
httpClients: httpClients, httpClients: httpClients,
} }
if len(observers) > 0 {
service.billingMetrics = observers[0]
}
return service
}
func (s *Service) observeBillingEvent(event string) {
if s.billingMetrics != nil {
s.billingMetrics.ObserveBillingEvent(event)
}
} }
func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) { func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) {
@@ -87,6 +116,20 @@ func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, use
} }
func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) { func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) {
return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString())
}
func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta, executionToken string) (Result, error) {
wasRunning := task.Status == "running"
claimed, err := s.store.ClaimTaskExecution(ctx, task.ID, executionToken, taskExecutionLeaseTTL)
if err != nil {
return Result{}, err
}
task = claimed
executionCtx, stopExecution := context.WithCancel(ctx)
defer stopExecution()
go s.renewTaskExecutionLease(executionCtx, stopExecution, task.ID, task.ExecutionToken)
ctx = executionCtx
executeStartedAt := time.Now() executeStartedAt := time.Now()
restoredRequest, err := s.restoreTaskRequestReferences(ctx, task) restoredRequest, err := s.restoreTaskRequestReferences(ctx, task)
if err != nil { if err != nil {
@@ -95,10 +138,10 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
body := normalizeRequest(task.Kind, restoredRequest) body := normalizeRequest(task.Kind, restoredRequest)
responseExecution := responseExecutionContext{} responseExecution := responseExecutionContext{}
modelType := modelTypeFromKind(task.Kind, body) modelType := modelTypeFromKind(task.Kind, body)
if err := s.store.MarkTaskRunning(ctx, task.ID, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil {
return Result{}, err return Result{}, err
} }
if task.Status != "running" { if !wasRunning {
if err := s.emit(ctx, task.ID, "task.running", "running", "starting", 0.12, "task pulled from queue and started", map[string]any{"modelType": modelType}, task.RunMode == "simulation"); err != nil { if err := s.emit(ctx, task.ID, "task.running", "running", "starting", 0.12, "task pulled from queue and started", map[string]any{"modelType": modelType}, task.RunMode == "simulation"); err != nil {
return Result{}, err return Result{}, err
} }
@@ -115,7 +158,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Reason: "request_validation_failed", Reason: "request_validation_failed",
ModelType: modelType, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, "bad_request", err.Error(), task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "bad_request", err.Error(), task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -135,14 +178,14 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Reason: "cloned_voice_binding_failed", Reason: "cloned_voice_binding_failed",
ModelType: modelType, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
return Result{Task: failed, Output: failed.Result}, err return Result{Task: failed, Output: failed.Result}, err
} }
if clonedVoice.Found { if clonedVoice.Found {
if err := s.store.MarkTaskRunning(ctx, task.ID, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil {
return Result{}, err return Result{}, err
} }
} }
@@ -151,7 +194,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
if err != nil { if err != nil {
code, message := responseExecutionFailure(err) code, message := responseExecutionFailure(err)
s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_chain", Reason: code, ModelType: modelType}) s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_chain", Reason: code, ModelType: modelType})
failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -183,7 +226,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Reason: "candidate_selection_failed", Reason: "candidate_selection_failed",
ModelType: modelType, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -202,7 +245,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Reason: store.ModelCandidateErrorCode(err), Reason: store.ModelCandidateErrorCode(err),
ModelType: modelType, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -224,7 +267,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
ExtraMetrics: []map[string]any{candidateFilterMetrics}, ExtraMetrics: []map[string]any{candidateFilterMetrics},
ModelType: modelType, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -240,7 +283,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Simulated: task.RunMode == "simulation", Scope: "candidate_output_token_filter", Reason: store.ModelCandidateErrorCode(err), Simulated: task.RunMode == "simulation", Scope: "candidate_output_token_filter", Reason: store.ModelCandidateErrorCode(err),
ExtraMetrics: []map[string]any{candidateFilterMetrics}, ModelType: modelType, ExtraMetrics: []map[string]any{candidateFilterMetrics}, ModelType: modelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -251,13 +294,120 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
if err != nil { if err != nil {
code, message := responseExecutionFailure(err) code, message := responseExecutionFailure(err)
s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_protocol", Reason: code, ModelType: modelType}) s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_protocol", Reason: code, ModelType: modelType})
failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", err)
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
return Result{Task: failed, Output: failed.Result}, err return Result{Task: failed, Output: failed.Result}, err
} }
} }
pricingByCandidate := map[string]resolvedPricing{}
preprocessingByCandidate := map[string]parameterPreprocessResult{}
reservationBillings := []any(nil)
reservationPricingSnapshot := map[string]any(nil)
if task.RunMode == "production" {
billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode)
if billingMode == "hold" {
holdErr := &clients.ClientError{Code: "billing_hold", Message: "production billing is temporarily on hold", Retryable: true}
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "billing_hold", holdErr.Error(), false, holdErr)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, holdErr
}
estimates := make([]candidateEstimate, 0, len(candidates))
var pricingErr error
var preprocessingErr error
var preprocessingCandidate store.RuntimeModelCandidate
for _, candidate := range candidates {
preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate)
preprocessingByCandidate[pricingCandidateKey(candidate)] = preprocessing
if preprocessing.Err != nil {
preprocessingErr = parameterPreprocessClientError(preprocessing.Err)
preprocessingCandidate = candidate
break
}
estimate, estimateErr := s.estimateCandidateV2(ctx, user, task.Kind, preprocessing.Body, candidate)
if estimateErr != nil {
pricingErr = estimateErr
if billingMode == "enforce" {
break
}
continue
}
estimates = append(estimates, estimate)
pricingByCandidate[pricingCandidateKey(candidate)] = estimate.Pricing
}
if preprocessingErr != nil {
preprocessing := preprocessingByCandidate[pricingCandidateKey(preprocessingCandidate)]
s.recordFailedAttempt(ctx, failedAttemptRecord{
Task: task, Body: preprocessing.Body, Candidate: &preprocessingCandidate,
AttemptNo: task.AttemptCount + 1, Code: clients.ErrorCode(preprocessingErr), Cause: preprocessingErr,
Simulated: false, Scope: "parameter_preprocessing", Reason: "parameter_preprocessing_failed",
ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)}, Preprocessing: &preprocessing.Log,
ModelType: preprocessingCandidate.ModelType,
})
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(preprocessingErr), preprocessingErr.Error(), false, preprocessingErr, parameterPreprocessingMetrics(preprocessing.Log))
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, preprocessingErr
}
if billingMode == "observe" {
legacyItems, legacyAmount := s.maximumLegacyCandidateEstimate(ctx, user, task.Kind, body, candidates, preprocessingByCandidate)
reservationBillings = legacyItems
candidateSnapshots := make([]any, 0, len(estimates))
for _, estimate := range estimates {
candidateSnapshots = append(candidateSnapshots, estimate.Snapshot)
}
observedAmount := ""
if maximumEstimate, estimateErr := maximumCandidateEstimate(estimates); estimateErr == nil {
observedAmount = maximumEstimate.Amount.String()
}
reservationPricingSnapshot = map[string]any{
"pricingVersion": "legacy-observe", "observedPricingVersion": pricingVersionV2,
"requestFingerprint": pricingRequestFingerprint(task.Kind, task.Model, body),
"reservationAmount": legacyAmount.String(), "observedReservationAmount": observedAmount,
"candidateCount": len(candidates), "observedCandidates": candidateSnapshots,
}
s.logger.Info("billing observe comparison", "taskID", task.ID, "legacyAmount", legacyAmount.String(), "v2Amount", observedAmount, "pricedCandidates", len(estimates), "candidateCount", len(candidates))
} else if pricingErr != nil || len(estimates) != len(candidates) {
if pricingErr == nil {
pricingErr = &PricingUnavailableError{Reason: "not every candidate has effective pricing"}
}
s.observeBillingEvent("pricing_unavailable")
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "pricing_unavailable", pricingErr.Error(), false, pricingErr)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, pricingErr
} else {
maximumEstimate, estimateErr := maximumCandidateEstimate(estimates)
if estimateErr != nil {
failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "pricing_unavailable", estimateErr.Error(), false, estimateErr)
if finishErr != nil {
return Result{}, finishErr
}
return Result{Task: failed, Output: failed.Result}, estimateErr
}
pricedCandidates := make([]store.RuntimeModelCandidate, 0, len(estimates))
candidateSnapshots := make([]any, 0, len(estimates))
for _, candidate := range candidates {
if _, ok := pricingByCandidate[pricingCandidateKey(candidate)]; ok {
pricedCandidates = append(pricedCandidates, candidate)
}
}
for _, estimate := range estimates {
candidateSnapshots = append(candidateSnapshots, estimate.Snapshot)
}
candidates = pricedCandidates
reservationBillings = maximumEstimate.Items
reservationPricingSnapshot = map[string]any{
"pricingVersion": pricingVersionV2, "requestFingerprint": pricingRequestFingerprint(task.Kind, task.Model, body),
"reservationAmount": maximumEstimate.Amount.String(), "candidateCount": len(estimates), "candidates": candidateSnapshots,
}
}
}
firstCandidateBody := body firstCandidateBody := body
normalizedModelType := modelType normalizedModelType := modelType
attemptNo := task.AttemptCount attemptNo := task.AttemptCount
@@ -270,7 +420,10 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
} }
}() }()
if len(candidates) > 0 { if len(candidates) > 0 {
preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidates[0]) preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidates[0])]
if !ok {
preprocessing = s.preprocessRequestWithScripts(ctx, task.Kind, body, candidates[0])
}
firstCandidateBody = preprocessing.Body firstCandidateBody = preprocessing.Body
firstPreprocessing = preprocessing.Log firstPreprocessing = preprocessing.Log
normalizedModelType = candidates[0].ModelType normalizedModelType = candidates[0].ModelType
@@ -290,18 +443,17 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Preprocessing: &firstPreprocessing, Preprocessing: &firstPreprocessing,
ModelType: normalizedModelType, ModelType: normalizedModelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, clients.ErrorCode(clientErr), clientErr.Error(), task.RunMode == "simulation", clientErr, parameterPreprocessingMetrics(firstPreprocessing)) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(clientErr), clientErr.Error(), task.RunMode == "simulation", clientErr, parameterPreprocessingMetrics(firstPreprocessing))
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
return Result{Task: failed, Output: failed.Result}, clientErr return Result{Task: failed, Output: failed.Result}, clientErr
} }
if err := s.store.MarkTaskRunning(ctx, task.ID, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil { if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil {
return Result{}, err return Result{}, err
} }
estimatedBillings := s.estimatedBillings(ctx, user, task.Kind, firstCandidateBody, candidates[0])
var reserveErr error var reserveErr error
walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, estimatedBillings) walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings, reservationPricingSnapshot)
if reserveErr != nil { if reserveErr != nil {
if errors.Is(reserveErr, store.ErrInsufficientWalletBalance) { if errors.Is(reserveErr, store.ErrInsufficientWalletBalance) {
attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{ attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{
@@ -318,7 +470,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut
Preprocessing: &firstPreprocessing, Preprocessing: &firstPreprocessing,
ModelType: normalizedModelType, ModelType: normalizedModelType,
}) })
failed, finishErr := s.failTask(ctx, task.ID, "insufficient_balance", reserveErr.Error(), task.RunMode == "simulation", reserveErr, parameterPreprocessingMetrics(firstPreprocessing)) failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "insufficient_balance", reserveErr.Error(), task.RunMode == "simulation", reserveErr, parameterPreprocessingMetrics(firstPreprocessing))
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
@@ -347,7 +499,10 @@ candidatesLoop:
var candidateErr error var candidateErr error
for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ { for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ {
nextAttemptNo := attemptNo + 1 nextAttemptNo := attemptNo + 1
preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate) preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)]
if !ok {
preprocessing = s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate)
}
preprocessingLog := preprocessing.Log preprocessingLog := preprocessing.Log
lastPreprocessing = &preprocessingLog lastPreprocessing = &preprocessingLog
if preprocessing.Err != nil { if preprocessing.Err != nil {
@@ -369,47 +524,80 @@ candidatesLoop:
break candidatesLoop break candidatesLoop
} }
candidateBody := preprocessing.Body candidateBody := preprocessing.Body
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record) candidatePricing := pricingByCandidate[pricingCandidateKey(candidate)]
response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record)
if err == nil { if err == nil {
attemptNo = nextAttemptNo attemptNo = nextAttemptNo
billings := s.billings(ctx, user, task.Kind, candidateBody, candidate, response, isSimulation(task, candidate)) var billings []any
finalAmount := fixedAmount(0)
finalAmountText := ""
pricingSnapshot := map[string]any{"pricingVersion": pricingVersionV2, "source": "simulation"}
if task.RunMode == "production" {
billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode)
var billingErr error
if billingMode == "observe" {
billings = s.billings(ctx, user, task.Kind, candidateBody, candidate, response, false)
finalAmount = billingItemsFixedTotal(billings)
pricingSnapshot = map[string]any{
"pricingVersion": "legacy-observe", "observedPricingVersion": pricingVersionV2,
"observedPricing": candidatePricing.Snapshot,
}
} else {
billings, finalAmount, _, billingErr = s.billingsWithResolvedPricingV2(ctx, user, task.Kind, candidateBody, candidate, response, false, candidatePricing)
pricingSnapshot = candidatePricing.Snapshot
}
if billingErr != nil {
review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: response.AttemptID, TaskStatus: "succeeded",
Code: "billing_calculation_failed", Message: "billing calculation requires manual review",
Result: response.Result, RequestID: response.RequestID,
PricingSnapshot: candidatePricing.Snapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
ResponseStartedAt: response.ResponseStartedAt, ResponseFinishedAt: response.ResponseFinishedAt,
ResponseDurationMS: response.ResponseDurationMS,
})
if reviewErr != nil {
return Result{}, reviewErr
}
walletReservationFinalized = true
s.logger.Warn("task succeeded but billing requires manual review", "taskID", task.ID, "error_category", "billing_calculation_failed")
return Result{Task: review, Output: response.Result}, nil
}
finalAmountText = finalAmount.String()
} else {
billings = s.billings(ctx, user, task.Kind, candidateBody, candidate, response, true)
}
record := buildSuccessRecord(task, user, candidateBody, candidate, response, billings, isSimulation(task, candidate)) record := buildSuccessRecord(task, user, candidateBody, candidate, response, billings, isSimulation(task, candidate))
record.Metrics = mergeMetrics(record.Metrics, candidateCapabilityFilterMetrics(candidateFilterSummary)) record.Metrics = mergeMetrics(record.Metrics, candidateCapabilityFilterMetrics(candidateFilterSummary))
record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log)) record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log))
record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics) record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics)
finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{ finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{
TaskID: task.ID, TaskID: task.ID,
Result: response.Result, ExecutionToken: task.ExecutionToken,
Billings: billings, AttemptID: response.AttemptID,
RequestID: record.RequestID, Result: response.Result,
ResolvedModel: record.ResolvedModel, Billings: billings,
Usage: record.Usage, RequestID: record.RequestID,
Metrics: record.Metrics, ResolvedModel: record.ResolvedModel,
BillingSummary: record.BillingSummary, Usage: record.Usage,
FinalChargeAmount: record.FinalChargeAmount, Metrics: record.Metrics,
ResponseStartedAt: record.ResponseStartedAt, BillingSummary: record.BillingSummary,
ResponseFinishedAt: record.ResponseFinishedAt, FinalChargeAmount: record.FinalChargeAmount,
ResponseDurationMS: record.ResponseDurationMS, FinalChargeAmountText: finalAmountText,
BillingCurrency: stringFromAny(record.BillingSummary["currency"]),
PricingSnapshot: pricingSnapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
ResponseStartedAt: record.ResponseStartedAt,
ResponseFinishedAt: record.ResponseFinishedAt,
ResponseDurationMS: record.ResponseDurationMS,
}) })
if finishErr != nil { if finishErr != nil {
return Result{}, finishErr return Result{}, finishErr
} }
if finished.FinalChargeAmount > 0 {
walletReservationFinalized = true
if settleErr := s.store.SettleTaskBilling(ctx, finished); settleErr != nil {
return Result{}, settleErr
}
} else if len(walletReservations) > 0 {
if releaseErr := s.store.ReleaseTaskBillingReservations(ctx, walletReservations, "task_billing_zero"); releaseErr != nil {
return Result{}, releaseErr
}
walletReservationFinalized = true
}
walletReservationFinalized = true walletReservationFinalized = true
if finished.FinalChargeAmount > 0 { if finished.BillingStatus == "pending" {
if err := s.emit(ctx, task.ID, "task.billing.settled", "succeeded", "billing", 0.98, "task billing settled", map[string]any{ if err := s.emit(ctx, task.ID, "task.billing.pending", "succeeded", "billing", 0.98, "task billing queued", map[string]any{
"amount": finished.FinalChargeAmount, "amount": finished.FinalChargeAmount, "currency": finished.BillingCurrency,
"currency": stringFromAny(record.BillingSummary["currency"]),
}, isSimulation(task, candidate)); err != nil { }, isSimulation(task, candidate)); err != nil {
return Result{}, err return Result{}, err
} }
@@ -426,6 +614,21 @@ candidatesLoop:
} }
return Result{Task: finished, Output: response.Result}, nil return Result{Task: finished, Output: response.Result}, nil
} }
var submissionUnknown *upstreamSubmissionUnknownError
if errors.As(err, &submissionUnknown) {
review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{
TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: submissionUnknown.AttemptID, TaskStatus: "failed",
Code: "upstream_submission_unknown", Message: submissionUnknown.Error(),
PricingSnapshot: candidatePricing.Snapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody),
})
if reviewErr != nil {
return Result{}, reviewErr
}
walletReservationFinalized = true
s.logger.Warn("upstream submission requires manual review", "taskID", task.ID, "attemptID", submissionUnknown.AttemptID, "error_category", "upstream_submission_unknown")
return Result{Task: review, Output: review.Result}, submissionUnknown
}
if isLocalRateLimitError(err) { if isLocalRateLimitError(err) {
lastErr = err lastErr = err
candidateErr = err candidateErr = err
@@ -573,14 +776,66 @@ candidatesLoop:
if lastPreprocessing != nil { if lastPreprocessing != nil {
extraMetrics = append(extraMetrics, parameterPreprocessingMetrics(*lastPreprocessing)) extraMetrics = append(extraMetrics, parameterPreprocessingMetrics(*lastPreprocessing))
} }
failed, err := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", lastErr, extraMetrics...) failed, err := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", lastErr, extraMetrics...)
if err != nil { if err != nil {
return Result{}, err return Result{}, err
} }
// FinishTaskFailure atomically transfers ownership of any reservation to the release Outbox.
walletReservationFinalized = true
return Result{Task: failed, Output: failed.Result}, lastErr return Result{Task: failed, Output: failed.Result}, lastErr
} }
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) { func pricingCandidateKey(candidate store.RuntimeModelCandidate) string {
return firstNonEmptyString(candidate.PlatformModelID, candidate.PlatformID+":"+candidate.ModelName)
}
func normalizedBillingEngineMode(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "enforce", "hold":
return strings.ToLower(strings.TrimSpace(value))
default:
return "observe"
}
}
func (s *Service) maximumLegacyCandidateEstimate(ctx context.Context, user *auth.User, kind string, body map[string]any, candidates []store.RuntimeModelCandidate, preprocessingByCandidate map[string]parameterPreprocessResult) ([]any, fixedAmount) {
var maximumItems []any
maximumAmount := fixedAmount(0)
for index, candidate := range candidates {
candidateBody := preprocessRequest(kind, cloneMap(body), candidate)
if preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)]; ok && preprocessing.Err == nil {
candidateBody = preprocessing.Body
}
items := s.estimatedBillings(ctx, user, kind, candidateBody, candidate)
amount := billingItemsFixedTotal(items)
if index == 0 || amount > maximumAmount {
maximumItems = items
maximumAmount = amount
}
}
return maximumItems, maximumAmount
}
func billingItemsFixedTotal(items []any) fixedAmount {
total := fixedAmount(0)
for _, raw := range items {
line, _ := raw.(map[string]any)
if line == nil {
continue
}
amount, err := fixedAmountFromAny(line["amount"])
if err == nil && amount > 0 {
next, addErr := addFixedAmounts(total, amount)
if addErr != nil {
return maxFixedAmount
}
total = next
}
}
return total
}
func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) {
simulated := isSimulation(task, candidate) simulated := isSimulation(task, candidate)
baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing)) baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing))
reservations := s.rateLimitReservations(ctx, user, candidate, body) reservations := s.rateLimitReservations(ctx, user, candidate, body)
@@ -599,16 +854,18 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.LeaseIDs) defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.LeaseIDs)
attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{ attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{
TaskID: task.ID, TaskID: task.ID,
AttemptNo: attemptNo, AttemptNo: attemptNo,
PlatformID: candidate.PlatformID, PlatformID: candidate.PlatformID,
PlatformModelID: candidate.PlatformModelID, PlatformModelID: candidate.PlatformModelID,
ClientID: candidate.ClientID, ClientID: candidate.ClientID,
QueueKey: candidate.QueueKey, QueueKey: candidate.QueueKey,
Status: "running", Status: "running",
Simulated: simulated, Simulated: simulated,
RequestSnapshot: s.slimTaskRequestSnapshot(task, body), RequestSnapshot: s.slimTaskRequestSnapshot(task, body),
Metrics: baseAttemptMetrics, Metrics: baseAttemptMetrics,
PricingSnapshot: pricing.Snapshot,
RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, body),
}) })
if err != nil { if err != nil {
return clients.Response{}, fmt.Errorf("create task attempt: %w", err) return clients.Response{}, fmt.Errorf("create task attempt: %w", err)
@@ -678,6 +935,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
publicResponseID = responseExecution.PublicResponseID publicResponseID = responseExecution.PublicResponseID
publicPreviousResponseID = responseExecution.PublicPreviousResponseID publicPreviousResponseID = responseExecution.PublicPreviousResponseID
} }
if err := s.store.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil {
return clients.Response{}, fmt.Errorf("mark upstream submission: %w", err)
}
response, err := client.Run(ctx, clients.Request{ response, err := client.Run(ctx, clients.Request{
Kind: task.Kind, Kind: task.Kind,
ModelType: candidate.ModelType, ModelType: candidate.ModelType,
@@ -691,7 +951,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
if strings.TrimSpace(remoteTaskID) == "" { if strings.TrimSpace(remoteTaskID) == "" {
return nil return nil
} }
return s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, attemptID, remoteTaskID, payload) return s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload)
}, },
Stream: boolFromMap(providerBody, "stream"), Stream: boolFromMap(providerBody, "stream"),
StreamDelta: onDelta, StreamDelta: onDelta,
@@ -702,6 +962,11 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
PreviousResponseTurns: responseExecution.PreviousTurns, PreviousResponseTurns: responseExecution.PreviousTurns,
}) })
callFinishedAt := time.Now() callFinishedAt := time.Now()
if err == nil {
if markErr := s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received"); markErr != nil {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
}
}
if response.ResponseStartedAt.IsZero() { if response.ResponseStartedAt.IsZero() {
response.ResponseStartedAt = callStartedAt response.ResponseStartedAt = callStartedAt
} }
@@ -715,6 +980,11 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
} }
} }
if err != nil { if err != nil {
if clients.ErrorResponseMetadata(err).StatusCode > 0 {
if markErr := s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received"); markErr != nil {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr}
}
}
retryable := clients.IsRetryable(err) retryable := clients.IsRetryable(err)
requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated) requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated)
if responseStartedAt.IsZero() { if responseStartedAt.IsZero() {
@@ -743,6 +1013,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
ErrorMessage: err.Error(), ErrorMessage: err.Error(),
}) })
_ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated) _ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated)
if !simulated && clients.ErrorResponseMetadata(err).StatusCode == 0 {
return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err}
}
s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected) s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected)
return clients.Response{}, err return clients.Response{}, err
} }
@@ -824,19 +1097,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user
return clients.Response{}, fmt.Errorf("commit rate limit reservations: %w", err) return clients.Response{}, fmt.Errorf("commit rate limit reservations: %w", err)
} }
rateReservationsFinalized = true rateReservationsFinalized = true
if err := s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{ response.AttemptID = attemptID
AttemptID: attemptID,
Status: "succeeded",
RequestID: response.RequestID,
Usage: usageToMap(response.Usage),
Metrics: mergeMetrics(taskMetrics(task, user, body, candidate, response, simulated), parameterPreprocessingMetrics(preprocessing)),
ResponseSnapshot: response.Result,
ResponseStartedAt: response.ResponseStartedAt,
ResponseFinishedAt: response.ResponseFinishedAt,
ResponseDurationMS: response.ResponseDurationMS,
}); err != nil {
return clients.Response{}, fmt.Errorf("finish task attempt: %w", err)
}
if err := s.store.RecordCacheAffinityObservation(context.WithoutCancel(ctx), store.CacheAffinityObservationInput{ if err := s.store.RecordCacheAffinityObservation(context.WithoutCancel(ctx), store.CacheAffinityObservationInput{
CacheAffinityKey: candidate.CacheAffinity.Key, CacheAffinityKey: candidate.CacheAffinity.Key,
CacheAffinityKeys: cacheAffinityRecordKeys, CacheAffinityKeys: cacheAffinityRecordKeys,
@@ -904,7 +1165,7 @@ func (s *Service) clientFor(candidate store.RuntimeModelCandidate, simulated boo
return s.clients["openai"] return s.clients["openai"]
} }
func (s *Service) failTask(ctx context.Context, taskID string, code string, message string, simulated bool, cause error, extraMetrics ...map[string]any) (store.GatewayTask, error) { func (s *Service) failTask(ctx context.Context, taskID string, executionToken string, code string, message string, simulated bool, cause error, extraMetrics ...map[string]any) (store.GatewayTask, error) {
requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(cause, simulated) requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(cause, simulated)
if len(extraMetrics) > 0 { if len(extraMetrics) > 0 {
values := append([]map[string]any{metrics}, extraMetrics...) values := append([]map[string]any{metrics}, extraMetrics...)
@@ -913,6 +1174,7 @@ func (s *Service) failTask(ctx context.Context, taskID string, code string, mess
metrics = s.withAttemptHistory(ctx, taskID, metrics) metrics = s.withAttemptHistory(ctx, taskID, metrics)
failed, err := s.store.FinishTaskFailure(ctx, store.FinishTaskFailureInput{ failed, err := s.store.FinishTaskFailure(ctx, store.FinishTaskFailureInput{
TaskID: taskID, TaskID: taskID,
ExecutionToken: executionToken,
Code: code, Code: code,
Message: message, Message: message,
Result: buildFailureResult(code, message, requestID, cause), Result: buildFailureResult(code, message, requestID, cause),
@@ -1041,7 +1303,7 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
if delay <= 0 { if delay <= 0 {
delay = 5 * time.Second delay = 5 * time.Second
} }
queued, err := s.store.RequeueTask(ctx, task.ID, delay, candidate.QueueKey) queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, delay, candidate.QueueKey)
if err != nil { if err != nil {
return store.GatewayTask{}, 0, err return store.GatewayTask{}, 0, err
} }
@@ -1057,7 +1319,7 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway
} }
func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) { func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) {
queued, err := s.store.RequeueTask(ctx, task.ID, 0, "") queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "")
if err != nil { if err != nil {
return store.GatewayTask{}, err return store.GatewayTask{}, err
} }
apps/api/internal/runner/wallet_execute_test.go
+14
View File
@@ -182,6 +182,20 @@ func TestExecuteWithMockClientRejectsConcurrentTasksBeyondWalletBalance(t *testi
if got := mockClient.calls.Load(); got != 1 { if got := mockClient.calls.Load(); got != 1 {
t.Fatalf("mock client calls = %d, want 1", got) t.Fatalf("mock client calls = %d, want 1", got)
} }
settlements, err := db.ClaimBillingSettlements(ctx, "wallet-execute-test", store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout)
if err != nil {
t.Fatalf("claim billing settlements: %v", err)
}
processed := 0
for _, settlement := range settlements {
if err := db.ProcessBillingSettlement(ctx, settlement); err != nil {
t.Fatalf("process billing settlement %s: %v", settlement.ID, err)
}
processed++
}
if processed != 1 {
t.Fatalf("processed billing settlements = %d, want 1", processed)
}
summary, err := db.GetWalletSummary(ctx, user, "resource") summary, err := db.GetWalletSummary(ctx, user, "resource")
if err != nil { if err != nil {
apps/api/internal/securityevents/metrics.go
+72 -16
View File
@@ -18,25 +18,32 @@ type MetricsSnapshotProvider interface {
type DynamicMetricsSnapshotProvider interface { type DynamicMetricsSnapshotProvider interface {
MetricsSnapshotProvider MetricsSnapshotProvider
SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) SecurityEventConnection(context.Context) (store.SecurityEventConnection, error)
BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error)
} }
type Metrics struct { type Metrics struct {
accepted atomic.Uint64 accepted atomic.Uint64
rejected atomic.Uint64 rejected atomic.Uint64
duplicate atomic.Uint64 duplicate atomic.Uint64
sessionsDeleted atomic.Uint64 sessionsDeleted atomic.Uint64
watermarkRejected atomic.Uint64 watermarkRejected atomic.Uint64
verificationAccepted atomic.Uint64 verificationAccepted atomic.Uint64
heartbeatAccepted atomic.Uint64 heartbeatAccepted atomic.Uint64
heartbeatFailed atomic.Uint64 heartbeatFailed atomic.Uint64
introspectionActive atomic.Uint64 introspectionActive atomic.Uint64
introspectionInactive atomic.Uint64 introspectionInactive atomic.Uint64
introspectionFailed atomic.Uint64 introspectionFailed atomic.Uint64
jwksSSFFailed atomic.Uint64 jwksSSFFailed atomic.Uint64
jwksOIDCFailed atomic.Uint64 jwksOIDCFailed atomic.Uint64
processingCount atomic.Uint64 processingCount atomic.Uint64
processingNanos atomic.Uint64 processingNanos atomic.Uint64
processingBuckets [6]atomic.Uint64 processingBuckets [6]atomic.Uint64
billingSettlementCompleted atomic.Uint64
billingSettlementRetry atomic.Uint64
billingManualReview atomic.Uint64
billingEstimateFailed atomic.Uint64
billingIdempotentReplay atomic.Uint64
billingPricingUnavailable atomic.Uint64
} }
var processingDurationBounds = [...]time.Duration{ var processingDurationBounds = [...]time.Duration{
@@ -99,6 +106,23 @@ func (m *Metrics) ObserveJWKSRefreshFailure(source string) {
m.jwksOIDCFailed.Add(1) m.jwksOIDCFailed.Add(1)
} }
func (m *Metrics) ObserveBillingEvent(event string) {
switch event {
case "settlement_completed":
m.billingSettlementCompleted.Add(1)
case "settlement_retry":
m.billingSettlementRetry.Add(1)
case "manual_review":
m.billingManualReview.Add(1)
case "estimate_failed":
m.billingEstimateFailed.Add(1)
case "idempotent_replay":
m.billingIdempotentReplay.Add(1)
case "pricing_unavailable":
m.billingPricingUnavailable.Add(1)
}
}
func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience string, enabled bool) http.Handler { func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience string, enabled bool) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mode := "disabled" mode := "disabled"
@@ -113,6 +137,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
return return
} }
} }
billing := store.BillingMetricsSnapshot{}
if billingProvider, ok := provider.(interface {
BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error)
}); ok {
var err error
billing, err = billingProvider.BillingMetrics(r.Context())
if err != nil {
http.Error(w, "metrics unavailable", http.StatusServiceUnavailable)
return
}
}
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{ outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{
{"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()}, {"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()},
@@ -156,6 +191,27 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str
} }
fmt.Fprintf(w, "easyai_gateway_ssf_mode{mode=\"%s\"} %d\n", modeName, value) fmt.Fprintf(w, "easyai_gateway_ssf_mode{mode=\"%s\"} %d\n", modeName, value)
} }
fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_backlog Current unsettled Outbox records.")
fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_backlog gauge")
fmt.Fprintf(w, "easyai_gateway_billing_settlement_backlog %d\n", billing.SettlementBacklog)
fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_delay_seconds Age of the oldest unsettled Outbox record.")
fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_delay_seconds gauge")
fmt.Fprintf(w, "easyai_gateway_billing_settlement_delay_seconds %.6f\n", billing.SettlementDelaySecs)
fmt.Fprintln(w, "# HELP easyai_gateway_billing_manual_review Current billing records requiring manual review.")
fmt.Fprintln(w, "# TYPE easyai_gateway_billing_manual_review gauge")
fmt.Fprintf(w, "easyai_gateway_billing_manual_review %d\n", billing.ManualReview)
fmt.Fprintln(w, "# HELP easyai_gateway_billing_orphan_frozen Wallets whose frozen amount differs from active reservations.")
fmt.Fprintln(w, "# TYPE easyai_gateway_billing_orphan_frozen gauge")
fmt.Fprintf(w, "easyai_gateway_billing_orphan_frozen %d\n", billing.OrphanFrozen)
fmt.Fprintln(w, "# HELP easyai_gateway_billing_pricing_unavailable_tasks Current tasks rejected because no effective price was available.")
fmt.Fprintln(w, "# TYPE easyai_gateway_billing_pricing_unavailable_tasks gauge")
fmt.Fprintf(w, "easyai_gateway_billing_pricing_unavailable_tasks %d\n", billing.PricingUnavailable)
plainCounter(w, "easyai_gateway_billing_settlements_completed_total", "Billing settlements completed by this process.", m.billingSettlementCompleted.Load())
plainCounter(w, "easyai_gateway_billing_settlement_retries_total", "Billing settlement retries scheduled by this process.", m.billingSettlementRetry.Load())
plainCounter(w, "easyai_gateway_billing_manual_review_transitions_total", "Billing records transitioned to manual review by this process.", m.billingManualReview.Load())
plainCounter(w, "easyai_gateway_billing_estimate_failures_total", "Pricing estimate requests that failed.", m.billingEstimateFailed.Load())
plainCounter(w, "easyai_gateway_billing_idempotent_replays_total", "Generation requests replayed idempotently.", m.billingIdempotentReplay.Load())
plainCounter(w, "easyai_gateway_billing_pricing_unavailable_total", "Pricing requests rejected because no effective price was available.", m.billingPricingUnavailable.Load())
}) })
} }
apps/api/internal/store/billing_settlements.go
+637
View File
@@ -0,0 +1,637 @@
package store
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
)
const (
BillingSettlementBatchSize = 50
BillingSettlementLockTimeout = 120 * time.Second
BillingSettlementMaxAttempts = 20
)
var ErrBillingSettlementNotRetryable = errors.New("billing settlement is not retryable")
type BillingSettlement struct {
ID string `json:"id"`
TaskID string `json:"taskId"`
Action string `json:"action"`
Amount float64 `json:"amount"`
Currency string `json:"currency"`
Status string `json:"status"`
Attempts int `json:"attempts"`
NextAttemptAt time.Time `json:"nextAttemptAt"`
LockedBy string `json:"lockedBy,omitempty"`
LockedAt string `json:"lockedAt,omitempty"`
LastErrorCode string `json:"lastErrorCode,omitempty"`
LastErrorMessage string `json:"lastErrorMessage,omitempty"`
ManualReviewReason string `json:"manualReviewReason,omitempty"`
PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
Payload map[string]any `json:"payload,omitempty"`
CompletedAt string `json:"completedAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
LockToken string `json:"-"`
amountExact string
}
type BillingSettlementListFilter struct {
Status string
Action string
Page int
PageSize int
}
type BillingSettlementListResult struct {
Items []BillingSettlement `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
}
type BillingMetricsSnapshot struct {
SettlementBacklog int64
SettlementDelaySecs float64
ManualReview int64
OrphanFrozen int64
PricingUnavailable int64
}
func (s *Store) BillingMetrics(ctx context.Context) (BillingMetricsSnapshot, error) {
var snapshot BillingMetricsSnapshot
err := s.pool.QueryRow(ctx, `
WITH open_outbox AS (
SELECT created_at
FROM settlement_outbox
WHERE status IN ('pending', 'processing', 'retryable_failed')
), active_reservations AS (
SELECT reserve.account_id, COALESCE(SUM(reserve.amount), 0) AS amount
FROM gateway_wallet_transactions reserve
WHERE reserve.transaction_type = 'reserve'
AND NOT EXISTS (
SELECT 1 FROM gateway_wallet_transactions release
WHERE release.account_id = reserve.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = reserve.idempotency_key || ':release'
)
GROUP BY reserve.account_id
)
SELECT
(SELECT COUNT(*) FROM open_outbox),
COALESCE((SELECT EXTRACT(EPOCH FROM now() - MIN(created_at)) FROM open_outbox), 0),
(SELECT COUNT(*) FROM gateway_tasks WHERE billing_status = 'manual_review'),
(SELECT COUNT(*) FROM gateway_wallet_accounts account
LEFT JOIN active_reservations reservation ON reservation.account_id = account.id
WHERE account.frozen_balance <> COALESCE(reservation.amount, 0)),
(SELECT COUNT(*) FROM gateway_tasks WHERE error_code = 'pricing_unavailable')`).Scan(
&snapshot.SettlementBacklog,
&snapshot.SettlementDelaySecs,
&snapshot.ManualReview,
&snapshot.OrphanFrozen,
&snapshot.PricingUnavailable,
)
return snapshot, err
}
func (s *Store) ClaimBillingSettlements(ctx context.Context, workerID string, limit int, staleAfter time.Duration) ([]BillingSettlement, error) {
if limit <= 0 || limit > BillingSettlementBatchSize {
limit = BillingSettlementBatchSize
}
if staleAfter <= 0 {
staleAfter = BillingSettlementLockTimeout
}
rows, err := s.pool.Query(ctx, `
WITH picked AS (
SELECT id
FROM settlement_outbox
WHERE (
status IN ('pending', 'retryable_failed')
AND next_attempt_at <= now()
)
OR (
status = 'processing'
AND locked_at < now() - ($3::int * interval '1 second')
)
ORDER BY next_attempt_at ASC, created_at ASC
LIMIT $2
FOR UPDATE SKIP LOCKED
)
UPDATE settlement_outbox outbox
SET status = 'processing',
attempts = outbox.attempts + 1,
locked_by = $1,
lock_token = gen_random_uuid(),
locked_at = now(),
updated_at = now()
FROM picked
WHERE outbox.id = picked.id
RETURNING outbox.id::text, outbox.task_id::text, outbox.action, outbox.amount::text,
outbox.currency, outbox.status, outbox.attempts, outbox.next_attempt_at,
COALESCE(outbox.locked_by, ''), COALESCE(outbox.lock_token::text, ''),
COALESCE(outbox.locked_at::text, ''), COALESCE(outbox.last_error_code, ''),
COALESCE(outbox.last_error_message, ''), COALESCE(outbox.manual_review_reason, ''),
outbox.pricing_snapshot, outbox.payload, COALESCE(outbox.completed_at::text, ''),
outbox.created_at, outbox.updated_at`,
workerID, limit, int(staleAfter/time.Second))
if err != nil {
return nil, err
}
defer rows.Close()
items := make([]BillingSettlement, 0)
for rows.Next() {
item, err := scanBillingSettlement(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Store) ProcessBillingSettlement(ctx context.Context, settlement BillingSettlement) error {
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var action string
var amount string
var currency string
var taskID string
var taskStatus string
var gatewayUserID string
var gatewayTenantID string
var userID string
var tenantID string
var tenantKey string
err := tx.QueryRow(ctx, `
SELECT outbox.action, outbox.amount::text, outbox.currency, outbox.task_id::text,
task.status, COALESCE(task.gateway_user_id::text, ''),
COALESCE(task.gateway_tenant_id::text, ''), task.user_id,
COALESCE(task.tenant_id, ''), COALESCE(task.tenant_key, '')
FROM settlement_outbox outbox
JOIN gateway_tasks task ON task.id = outbox.task_id
WHERE outbox.id = $1::uuid
AND outbox.status = 'processing'
AND outbox.lock_token = $2::uuid
FOR UPDATE OF outbox, task`, settlement.ID, settlement.LockToken).Scan(
&action, &amount, &currency, &taskID, &taskStatus, &gatewayUserID,
&gatewayTenantID, &userID, &tenantID, &tenantKey,
)
if err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET billing_status = 'processing', billing_updated_at = now(), updated_at = now()
WHERE id = $1::uuid`, taskID); err != nil {
return err
}
switch action {
case "settle":
if taskStatus != "succeeded" {
return fmt.Errorf("settle action requires succeeded task")
}
if err := settleBillingOutboxTx(ctx, tx, taskID, gatewayUserID, gatewayTenantID, userID, tenantID, tenantKey, currency, amount); err != nil {
return err
}
return completeBillingOutboxTx(ctx, tx, settlement.ID, settlement.LockToken, taskID, "settled")
case "release":
if err := releaseBillingOutboxTx(ctx, tx, taskID, gatewayUserID, gatewayTenantID, currency); err != nil {
return err
}
return completeBillingOutboxTx(ctx, tx, settlement.ID, settlement.LockToken, taskID, "released")
default:
return fmt.Errorf("unsupported billing settlement action %q", action)
}
})
}
func settleBillingOutboxTx(
ctx context.Context,
tx pgx.Tx,
taskID string,
gatewayUserID string,
gatewayTenantID string,
userID string,
tenantID string,
tenantKey string,
currency string,
amount string,
) error {
if gatewayUserID == "" {
return fmt.Errorf("task %s has no gateway wallet user", taskID)
}
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_accounts (
gateway_tenant_id, gateway_user_id, tenant_id, tenant_key, user_id, currency
)
VALUES (NULLIF($1, '')::uuid, $2::uuid, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), $6)
ON CONFLICT (gateway_user_id, currency) DO NOTHING`,
gatewayTenantID, gatewayUserID, tenantID, tenantKey, userID, currency); err != nil {
return err
}
if err := ensureWalletAccountAuditGuard(ctx, tx, gatewayUserID, currency); err != nil {
return err
}
var accountID string
var balanceBefore string
var frozenBefore string
if err := tx.QueryRow(ctx, `
SELECT id::text, balance::text, frozen_balance::text
FROM gateway_wallet_accounts
WHERE gateway_user_id = $1::uuid AND currency = $2
FOR UPDATE`, gatewayUserID, currency).Scan(&accountID, &balanceBefore, &frozenBefore); err != nil {
return err
}
var alreadySettled bool
if err := tx.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM gateway_wallet_transactions
WHERE account_id = $1::uuid AND idempotency_key = $2
)`, accountID, billingIdempotencyKey(taskID)).Scan(&alreadySettled); err != nil {
return err
}
if alreadySettled {
return nil
}
var reservationKey string
var reservedAmount string
if err := tx.QueryRow(ctx, `
SELECT COALESCE(reserve.idempotency_key, ''), reserve.amount::text
FROM gateway_wallet_transactions reserve
WHERE reserve.account_id = $1::uuid
AND reserve.reference_type = 'gateway_task'
AND reserve.reference_id = $2
AND reserve.transaction_type = 'reserve'
AND NOT EXISTS (
SELECT 1 FROM gateway_wallet_transactions release
WHERE release.account_id = reserve.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = reserve.idempotency_key || ':release'
)
ORDER BY reserve.created_at DESC
LIMIT 1`, accountID, taskID).Scan(&reservationKey, &reservedAmount); errors.Is(err, pgx.ErrNoRows) {
reservationKey = ""
reservedAmount = "0"
} else if err != nil {
return err
}
var balanceAfter string
var frozenAfter string
err := tx.QueryRow(ctx, `
UPDATE gateway_wallet_accounts
SET balance = balance - $2::numeric,
total_spent = total_spent + $2::numeric,
frozen_balance = GREATEST(0, frozen_balance - $3::numeric),
updated_at = now()
WHERE id = $1::uuid
AND balance - frozen_balance + $3::numeric >= $2::numeric
RETURNING balance::text, frozen_balance::text`,
accountID, amount, reservedAmount).Scan(&balanceAfter, &frozenAfter)
if errors.Is(err, pgx.ErrNoRows) {
return fmt.Errorf("%w: task %s settlement amount exceeds spendable balance", ErrInsufficientWalletBalance, taskID)
}
if err != nil {
return err
}
var reservedPositive bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, reservedAmount).Scan(&reservedPositive); err != nil {
return err
}
if reservedPositive {
releaseMetadata, _ := json.Marshal(map[string]any{
"taskId": taskID, "reason": "task_billing_settled",
"reserved": reservedAmount, "frozenBefore": frozenBefore, "frozenAfter": frozenAfter,
})
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata
)
VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release',
$4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb
)
ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`,
accountID, gatewayTenantID, gatewayUserID, reservedAmount, balanceBefore,
billingReservationReleaseIdempotencyKey(reservationKey), taskID, string(releaseMetadata)); err != nil {
return err
}
}
metadata, _ := json.Marshal(map[string]any{
"taskId": taskID, "reservedAmount": reservedAmount,
"frozenBefore": frozenBefore, "frozenAfter": frozenAfter,
})
_, err = tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata
)
VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'debit', 'task_billing',
$4::numeric, $5::numeric, $6::numeric, $7, 'gateway_task', $8, $9::jsonb
)
ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`,
accountID, gatewayTenantID, gatewayUserID, amount, balanceBefore, balanceAfter,
billingIdempotencyKey(taskID), taskID, string(metadata))
return err
}
func releaseBillingOutboxTx(ctx context.Context, tx pgx.Tx, taskID string, gatewayUserID string, gatewayTenantID string, currency string) error {
if gatewayUserID == "" {
return nil
}
var accountID string
var balance string
var frozenBefore string
err := tx.QueryRow(ctx, `
SELECT id::text, balance::text, frozen_balance::text
FROM gateway_wallet_accounts
WHERE gateway_user_id = $1::uuid AND currency = $2
FOR UPDATE`, gatewayUserID, currency).Scan(&accountID, &balance, &frozenBefore)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
var reservationKey string
var reservedAmount string
err = tx.QueryRow(ctx, `
SELECT COALESCE(reserve.idempotency_key, ''), reserve.amount::text
FROM gateway_wallet_transactions reserve
WHERE reserve.account_id = $1::uuid
AND reserve.reference_type = 'gateway_task'
AND reserve.reference_id = $2
AND reserve.transaction_type = 'reserve'
AND NOT EXISTS (
SELECT 1 FROM gateway_wallet_transactions release
WHERE release.account_id = reserve.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = reserve.idempotency_key || ':release'
)
ORDER BY reserve.created_at DESC
LIMIT 1`, accountID, taskID).Scan(&reservationKey, &reservedAmount)
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
if err != nil {
return err
}
var frozenAfter string
if err := tx.QueryRow(ctx, `
UPDATE gateway_wallet_accounts
SET frozen_balance = frozen_balance - $2::numeric,
updated_at = now()
WHERE id = $1::uuid
AND frozen_balance >= $2::numeric
RETURNING frozen_balance::text`, accountID, reservedAmount).Scan(&frozenAfter); err != nil {
return err
}
metadata, _ := json.Marshal(map[string]any{
"taskId": taskID, "reason": "task_terminal_release",
"reserved": reservedAmount, "frozenBefore": frozenBefore, "frozenAfter": frozenAfter,
})
_, err = tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata
)
VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release',
$4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb
)
ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`,
accountID, gatewayTenantID, gatewayUserID, reservedAmount, balance,
billingReservationReleaseIdempotencyKey(reservationKey), taskID, string(metadata))
return err
}
func completeBillingOutboxTx(ctx context.Context, tx pgx.Tx, settlementID string, lockToken string, taskID string, billingStatus string) error {
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET billing_status = $2,
reservation_amount = 0,
billing_updated_at = now(),
billing_settled_at = CASE WHEN $2 = 'settled' THEN now() ELSE billing_settled_at END,
updated_at = now()
WHERE id = $1::uuid`, taskID, billingStatus); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
SELECT task.id,
COALESCE((SELECT MAX(event.seq) + 1 FROM gateway_task_events event WHERE event.task_id = task.id), 1),
CASE WHEN $2 = 'settled' THEN 'task.billing.settled' ELSE 'task.billing.released' END,
task.status,
'billing',
1,
CASE WHEN $2 = 'settled' THEN 'task billing settled' ELSE 'task billing reservation released' END,
jsonb_build_object('settlementId', $3::text, 'billingStatus', $2::text),
task.run_mode = 'simulation'
FROM gateway_tasks task
WHERE task.id = $1::uuid`, taskID, billingStatus, settlementID); err != nil {
return err
}
tag, err := tx.Exec(ctx, `
UPDATE settlement_outbox
SET status = 'completed',
completed_at = now(),
locked_by = NULL,
lock_token = NULL,
locked_at = NULL,
last_error_code = NULL,
last_error_message = NULL,
updated_at = now()
WHERE id = $1::uuid
AND status = 'processing'
AND lock_token = $2::uuid`, settlementID, lockToken)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return pgx.ErrNoRows
}
return nil
}
func (s *Store) MarkBillingSettlementFailed(ctx context.Context, settlement BillingSettlement, code string, message string, delay time.Duration) error {
if delay < time.Second {
delay = time.Second
}
if delay > 15*time.Minute {
delay = 15 * time.Minute
}
manualReview := settlement.Attempts >= BillingSettlementMaxAttempts
status := "retryable_failed"
taskStatus := "retryable_failed"
manualReason := ""
if manualReview {
status = "manual_review"
taskStatus = "manual_review"
manualReason = "maximum settlement attempts exceeded"
}
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
UPDATE settlement_outbox
SET status = $3,
next_attempt_at = now() + ($4::int * interval '1 second'),
locked_by = NULL,
lock_token = NULL,
locked_at = NULL,
last_error_code = NULLIF($5, ''),
last_error_message = NULLIF($6, ''),
manual_review_reason = NULLIF($7, ''),
updated_at = now()
WHERE id = $1::uuid
AND lock_token = $2::uuid
AND status = 'processing'`,
settlement.ID, settlement.LockToken, status, int(delay/time.Second), code, message, manualReason)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return pgx.ErrNoRows
}
_, err = tx.Exec(ctx, `
UPDATE gateway_tasks
SET billing_status = $2,
billing_updated_at = now(),
updated_at = now()
WHERE id = $1::uuid`, settlement.TaskID, taskStatus)
return err
})
}
func (s *Store) ListBillingSettlements(ctx context.Context, filter BillingSettlementListFilter) (BillingSettlementListResult, error) {
page := filter.Page
if page <= 0 {
page = 1
}
pageSize := filter.PageSize
if pageSize <= 0 {
pageSize = 50
}
if pageSize > 100 {
pageSize = 100
}
status := strings.TrimSpace(filter.Status)
action := strings.TrimSpace(filter.Action)
var total int
if err := s.pool.QueryRow(ctx, `
SELECT count(*)
FROM settlement_outbox
WHERE (NULLIF($1, '') IS NULL OR status = $1)
AND (NULLIF($2, '') IS NULL OR action = $2)`, status, action).Scan(&total); err != nil {
return BillingSettlementListResult{}, err
}
rows, err := s.pool.Query(ctx, `
SELECT id::text, task_id::text, action, amount::text, currency, status, attempts,
next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''),
COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''),
COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''),
pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at
FROM settlement_outbox
WHERE (NULLIF($1, '') IS NULL OR status = $1)
AND (NULLIF($2, '') IS NULL OR action = $2)
ORDER BY created_at DESC
LIMIT $3 OFFSET $4`, status, action, pageSize, (page-1)*pageSize)
if err != nil {
return BillingSettlementListResult{}, err
}
defer rows.Close()
items := make([]BillingSettlement, 0)
for rows.Next() {
item, err := scanBillingSettlement(rows)
if err != nil {
return BillingSettlementListResult{}, err
}
items = append(items, item)
}
if err := rows.Err(); err != nil {
return BillingSettlementListResult{}, err
}
return BillingSettlementListResult{Items: items, Total: total, Page: page, PageSize: pageSize}, nil
}
func (s *Store) RetryBillingSettlementTx(ctx context.Context, tx Tx, id string, idempotencyKeyHash string) (BillingSettlement, bool, error) {
current, err := scanBillingSettlement(tx.QueryRow(ctx, `
SELECT id::text, task_id::text, action, amount::text, currency, status, attempts,
next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''),
COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''),
COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''),
pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at
FROM settlement_outbox
WHERE id = $1::uuid
FOR UPDATE`, id))
if err != nil {
return BillingSettlement{}, false, err
}
var recordedHash string
if err := tx.QueryRow(ctx, `
SELECT COALESCE(retry_idempotency_key_hash, '')
FROM settlement_outbox
WHERE id = $1::uuid`, id).Scan(&recordedHash); err != nil {
return BillingSettlement{}, false, err
}
if recordedHash != "" && recordedHash == idempotencyKeyHash {
return current, true, nil
}
if current.Status != "retryable_failed" && current.Status != "manual_review" {
return BillingSettlement{}, false, ErrBillingSettlementNotRetryable
}
item, err := scanBillingSettlement(tx.QueryRow(ctx, `
UPDATE settlement_outbox
SET status = 'pending',
next_attempt_at = now(),
locked_by = NULL,
lock_token = NULL,
locked_at = NULL,
manual_review_reason = NULL,
retry_idempotency_key_hash = $2,
retry_requested_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status IN ('retryable_failed', 'manual_review')
RETURNING id::text, task_id::text, action, amount::text, currency, status, attempts,
next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''),
COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''),
COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''),
pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at`, id, idempotencyKeyHash))
if err != nil {
return BillingSettlement{}, false, err
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET billing_status = 'pending', billing_updated_at = now(), updated_at = now()
WHERE id = $1::uuid`, item.TaskID); err != nil {
return BillingSettlement{}, false, err
}
return item, false, nil
}
func scanBillingSettlement(scanner taskScanner) (BillingSettlement, error) {
var item BillingSettlement
var pricingSnapshotBytes []byte
var payloadBytes []byte
if err := scanner.Scan(
&item.ID, &item.TaskID, &item.Action, &item.amountExact, &item.Currency,
&item.Status, &item.Attempts, &item.NextAttemptAt, &item.LockedBy, &item.LockToken,
&item.LockedAt, &item.LastErrorCode, &item.LastErrorMessage, &item.ManualReviewReason,
&pricingSnapshotBytes, &payloadBytes, &item.CompletedAt, &item.CreatedAt, &item.UpdatedAt,
); err != nil {
return BillingSettlement{}, err
}
item.Amount, _ = strconv.ParseFloat(item.amountExact, 64)
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
item.Payload = decodeObject(payloadBytes)
return item, nil
}
apps/api/internal/store/billing_v2_integration_test.go
+408
View File
@@ -0,0 +1,408 @@
package store
import (
"context"
"errors"
"os"
"strings"
"testing"
"time"
"github.com/easyai/easyai-ai-gateway/apps/api/internal/auth"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func TestTaskIdempotencyAndExecutionLease(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-v2-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID}
input := CreateTaskInput{
Kind: "images.generations", Model: "billing-v2-model", RunMode: "simulation",
Request: map[string]any{"model": "billing-v2-model", "prompt": "lease"},
IdempotencyKeyHash: "key-hash-" + uuid.NewString(), IdempotencyRequestHash: "request-a",
}
created, err := db.CreateTaskIdempotent(ctx, input, user)
if err != nil || created.Replayed {
t.Fatalf("create task result=%+v err=%v", created, err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.Task.ID)
})
replayed, err := db.CreateTaskIdempotent(ctx, input, user)
if err != nil || !replayed.Replayed || replayed.Task.ID != created.Task.ID {
t.Fatalf("replay result=%+v err=%v", replayed, err)
}
input.IdempotencyRequestHash = "request-b"
if _, err := db.CreateTaskIdempotent(ctx, input, user); !errors.Is(err, ErrIdempotencyKeyReused) {
t.Fatalf("different request error=%v", err)
}
firstToken := uuid.NewString()
claimed, err := db.ClaimTaskExecution(ctx, created.Task.ID, firstToken, 5*time.Minute)
if err != nil || claimed.ExecutionToken != firstToken {
t.Fatalf("first claim task=%+v err=%v", claimed, err)
}
if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, firstToken, 5*time.Minute); err != nil {
t.Fatalf("renew first lease: %v", err)
}
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.Task.ID); err != nil {
t.Fatal(err)
}
secondToken := uuid.NewString()
if _, err := db.ClaimTaskExecution(ctx, created.Task.ID, secondToken, 5*time.Minute); err != nil {
t.Fatalf("take over expired lease: %v", err)
}
if _, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: firstToken, Code: "old_worker", Message: "old"}); !errors.Is(err, ErrTaskExecutionLeaseLost) {
t.Fatalf("old worker terminal error=%v", err)
}
finished, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: secondToken, Code: "new_worker", Message: "new"})
if err != nil || finished.ErrorCode != "new_worker" {
t.Fatalf("new worker terminal task=%+v err=%v", finished, err)
}
if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, secondToken, 5*time.Minute); !errors.Is(err, ErrTaskExecutionFinished) {
t.Fatalf("terminal task renewal error=%v", err)
}
}
func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-review-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID}
created, err := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
Request: map[string]any{"model": "billing-v2-model"},
}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
})
firstToken := uuid.NewString()
if _, err := db.ClaimTaskExecution(ctx, created.ID, firstToken, 5*time.Minute); err != nil {
t.Fatal(err)
}
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{
TaskID: created.ID, AttemptNo: 1, Status: "running", RequestSnapshot: map[string]any{"model": created.Model},
})
if err != nil {
t.Fatal(err)
}
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil {
t.Fatal(err)
}
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil {
t.Fatal(err)
}
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); !errors.Is(err, ErrTaskExecutionManualReview) {
t.Fatalf("ambiguous submission takeover error=%v", err)
}
review, err := db.GetTask(ctx, created.ID)
if err != nil {
t.Fatal(err)
}
if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" {
t.Fatalf("manual review task=%+v", review)
}
var outboxStatus string
var outboxAction string
var reviewReason string
if err := db.pool.QueryRow(ctx, `
SELECT status, action, COALESCE(manual_review_reason, '')
FROM settlement_outbox
WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&outboxStatus, &outboxAction, &reviewReason); err != nil {
t.Fatal(err)
}
if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_submission_unknown" {
t.Fatalf("review outbox status=%s action=%s reason=%s", outboxStatus, outboxAction, reviewReason)
}
}
func TestExpiredExecutionLeaseCanResumeAfterKnownRejectedResponse(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
_, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-known-response-" + uuid.NewString(), GatewayUserID: gatewayUserID}
created, err := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
Request: map[string]any{"model": "billing-v2-model"},
}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
})
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
t.Fatal(err)
}
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"})
if err != nil {
t.Fatal(err)
}
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "response_received"); err != nil {
t.Fatal(err)
}
if err := db.FinishTaskAttempt(ctx, FinishTaskAttemptInput{AttemptID: attemptID, Status: "failed", ErrorCode: "upstream_rejected"}); err != nil {
t.Fatal(err)
}
if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil {
t.Fatal(err)
}
if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil {
t.Fatalf("known rejected response should remain retryable: %v", err)
}
}
func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
_, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-direct-review-" + uuid.NewString(), GatewayUserID: gatewayUserID}
created, err := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
Request: map[string]any{"model": "billing-v2-model"},
}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
})
token := uuid.NewString()
if _, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute); err != nil {
t.Fatal(err)
}
attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"})
if err != nil {
t.Fatal(err)
}
if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil {
t.Fatal(err)
}
if _, err := db.FinishTaskManualReview(ctx, FinishTaskManualReviewInput{
TaskID: created.ID, ExecutionToken: token, AttemptID: attemptID, TaskStatus: "failed",
Code: "upstream_submission_unknown", Message: "upstream submission result is unknown",
}); err != nil {
t.Fatal(err)
}
var visible bool
if err := db.pool.QueryRow(ctx, `
SELECT EXISTS (
SELECT 1 FROM settlement_outbox
WHERE task_id=$1::uuid AND status='manual_review'
AND action='release' AND manual_review_reason='upstream_submission_unknown'
)`, created.ID).Scan(&visible); err != nil {
t.Fatal(err)
}
if !visible {
t.Fatal("manual review billing record is not visible in settlement outbox")
}
}
func TestBillingSettlementStaleTakeoverDebitsExactlyOnce(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-settle-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID}
if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{GatewayUserID: gatewayUserID, Currency: "resource", Balance: 10, Reason: "billing v2 test"}); err != nil {
t.Fatal(err)
}
created, err := db.CreateTask(ctx, CreateTaskInput{Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", Request: map[string]any{"model": "billing-v2-model"}}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cleanupCtx := context.Background()
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid`, gatewayUserID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid`, gatewayUserID)
})
token := uuid.NewString()
claimed, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute)
if err != nil {
t.Fatal(err)
}
amount := "1.000000001"
reservations, err := db.ReserveTaskBilling(ctx, claimed, user, []any{map[string]any{"currency": "resource", "amount": amount}}, map[string]any{
"pricingVersion": "effective-pricing-v2", "reservationAmount": amount, "currency": "resource", "requestFingerprint": "billing-v2-test",
})
if err != nil || len(reservations) != 1 {
t.Fatalf("reserve=%+v err=%v", reservations, err)
}
if _, err := db.FinishTaskSuccess(ctx, FinishTaskSuccessInput{
TaskID: created.ID, ExecutionToken: token, Result: map[string]any{"ok": true},
FinalChargeAmountText: amount, BillingCurrency: "resource",
PricingSnapshot: map[string]any{"pricingVersion": "effective-pricing-v2"},
}); err != nil {
t.Fatal(err)
}
firstClaims, err := db.ClaimBillingSettlements(ctx, "worker-one", BillingSettlementBatchSize, BillingSettlementLockTimeout)
if err != nil {
t.Fatal(err)
}
first := settlementForTask(t, firstClaims, created.ID)
if _, err := db.pool.Exec(ctx, `UPDATE settlement_outbox SET locked_at=now()-interval '3 minutes' WHERE id=$1::uuid`, first.ID); err != nil {
t.Fatal(err)
}
secondClaims, err := db.ClaimBillingSettlements(ctx, "worker-two", BillingSettlementBatchSize, BillingSettlementLockTimeout)
if err != nil {
t.Fatal(err)
}
second := settlementForTask(t, secondClaims, created.ID)
if err := db.ProcessBillingSettlement(ctx, first); !errors.Is(err, pgx.ErrNoRows) {
t.Fatalf("stale settlement lock error=%v", err)
}
if err := db.ProcessBillingSettlement(ctx, second); err != nil {
t.Fatalf("process takeover: %v", err)
}
var walletExact bool
if err := db.pool.QueryRow(ctx, `
SELECT balance = 8.999999999::numeric
AND frozen_balance = 0::numeric
AND total_spent = 1.000000001::numeric
FROM gateway_wallet_accounts
WHERE gateway_user_id=$1::uuid AND currency='resource'`, gatewayUserID).Scan(&walletExact); err != nil {
t.Fatal(err)
}
if !walletExact {
t.Fatal("wallet amounts did not preserve nine-decimal settlement")
}
var billingTransactions int
if err := db.pool.QueryRow(ctx, `SELECT count(*) FROM gateway_wallet_transactions WHERE reference_id=$1 AND transaction_type='task_billing'`, created.ID).Scan(&billingTransactions); err != nil {
t.Fatal(err)
}
if billingTransactions != 1 {
t.Fatalf("task billing transactions=%d", billingTransactions)
}
if _, err := db.pool.Exec(ctx, `
DELETE FROM gateway_wallet_accounts
WHERE gateway_user_id=$1::uuid AND currency='resource'`, gatewayUserID); err == nil {
t.Fatal("wallet account with audit transactions must not be deletable")
}
if err := db.pool.QueryRow(ctx, `
SELECT count(*) FROM gateway_wallet_transactions
WHERE gateway_user_id=$1::uuid AND reference_id=$2`, gatewayUserID, created.ID).Scan(&billingTransactions); err != nil {
t.Fatal(err)
}
if billingTransactions == 0 {
t.Fatal("wallet audit transactions were lost after rejected account deletion")
}
settled, err := db.GetTask(ctx, created.ID)
if err != nil || settled.BillingStatus != "settled" {
t.Fatalf("settled task=%+v err=%v", settled, err)
}
}
func TestReleaseTaskBillingReservationsPreservesNineDecimalPlaces(t *testing.T) {
db := billingV2IntegrationStore(t)
ctx := context.Background()
tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db)
user := &auth.User{ID: "billing-release-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID}
if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{
GatewayUserID: gatewayUserID, Currency: "resource", Balance: 1, Reason: "billing v2 release test",
}); err != nil {
t.Fatal(err)
}
created, err := db.CreateTask(ctx, CreateTaskInput{
Kind: "images.generations", Model: "billing-v2-model", RunMode: "production",
Request: map[string]any{"model": "billing-v2-model"},
}, user)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
cleanupCtx := context.Background()
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid`, gatewayUserID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid`, gatewayUserID)
})
amount := "0.000000001"
reservations, err := db.ReserveTaskBilling(ctx, created, user, nil, map[string]any{
"pricingVersion": "effective-pricing-v2", "reservationAmount": amount, "currency": "resource",
})
if err != nil || len(reservations) != 1 {
t.Fatalf("reserve=%+v err=%v", reservations, err)
}
if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{
GatewayUserID: gatewayUserID, Currency: "resource", BalanceText: "0", Reason: "must not cross frozen balance",
}); !errors.Is(err, ErrBalanceBelowFrozen) {
t.Fatalf("balance below frozen error=%v", err)
}
if err := db.ReleaseTaskBillingReservations(ctx, reservations, "integration_test"); err != nil {
t.Fatal(err)
}
if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{
GatewayUserID: gatewayUserID, Currency: "resource", BalanceText: "0.123456789", Reason: "exact adjustment",
}); err != nil {
t.Fatal(err)
}
if _, err := db.RechargeUserWalletBalance(ctx, WalletRechargeInput{
GatewayUserID: gatewayUserID, Currency: "resource", AmountText: "0.000000001", Reason: "exact recharge",
}); err != nil {
t.Fatal(err)
}
var exact bool
if err := db.pool.QueryRow(ctx, `
SELECT account.balance = 0.123456790::numeric
AND account.frozen_balance = 0::numeric
AND task.reservation_amount = 0::numeric
AND task.billing_status = 'not_started'
AND EXISTS (
SELECT 1 FROM gateway_wallet_transactions transaction
WHERE transaction.reference_id = task.id::text
AND transaction.transaction_type = 'release'
AND transaction.amount = 0.000000001::numeric
)
FROM gateway_wallet_accounts account
JOIN gateway_tasks task ON task.gateway_user_id = account.gateway_user_id
WHERE task.id=$1::uuid AND account.currency='resource'`, created.ID).Scan(&exact); err != nil {
t.Fatal(err)
}
if !exact {
t.Fatal("nine-decimal reservation was not released exactly")
}
}
func billingV2IntegrationStore(t *testing.T) *Store {
t.Helper()
databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run billing v2 PostgreSQL integration tests")
}
db, err := Connect(context.Background(), databaseURL)
if err != nil {
t.Fatal(err)
}
var databaseName string
if err := db.pool.QueryRow(context.Background(), `SELECT current_database()`).Scan(&databaseName); err != nil {
db.Close()
t.Fatal(err)
}
if !strings.Contains(strings.ToLower(databaseName), "test") {
db.Close()
t.Fatalf("refusing to use non-test database %q", databaseName)
}
t.Cleanup(db.Close)
return db
}
func settlementForTask(t *testing.T, items []BillingSettlement, taskID string) BillingSettlement {
t.Helper()
for _, item := range items {
if item.TaskID == taskID {
return item
}
}
t.Fatalf("settlement for task %s not claimed", taskID)
return BillingSettlement{}
}
apps/api/internal/store/postgres.go
+196 -112
View File
@@ -47,16 +47,23 @@ func normalizeAPIKeyScopes(scopes []string) []string {
} }
var ( var (
ErrInvalidCredentials = errors.New("invalid account or password") ErrInvalidCredentials = errors.New("invalid account or password")
ErrInvalidInvitation = errors.New("invalid or expired invitation code") ErrInvalidInvitation = errors.New("invalid or expired invitation code")
ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty") ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty")
ErrAccessRuleResourceDenied = errors.New("access rule resource is not available") ErrAccessRuleResourceDenied = errors.New("access rule resource is not available")
ErrInsufficientWalletBalance = errors.New("insufficient wallet balance") ErrInsufficientWalletBalance = errors.New("insufficient wallet balance")
ErrLocalUserRequired = errors.New("local gateway user is required") ErrLocalUserRequired = errors.New("local gateway user is required")
ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged") ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged")
ErrProtectedDefault = errors.New("protected default resource cannot be deleted") ErrBalanceBelowFrozen = errors.New("wallet balance cannot be below frozen balance")
ErrUserAlreadyExists = errors.New("user already exists") ErrInvalidWalletAmount = errors.New("wallet amount must be a decimal with at most nine fractional digits")
ErrWeakPassword = errors.New("password must be at least 8 characters") ErrIdempotencyKeyReused = errors.New("idempotency key was reused for a different request")
ErrTaskExecutionLeaseUnavailable = errors.New("task execution lease is unavailable")
ErrTaskExecutionLeaseLost = errors.New("task execution lease was lost")
ErrTaskExecutionFinished = errors.New("task execution already finished")
ErrTaskExecutionManualReview = errors.New("task execution requires manual review")
ErrProtectedDefault = errors.New("protected default resource cannot be deleted")
ErrUserAlreadyExists = errors.New("user already exists")
ErrWeakPassword = errors.New("password must be at least 8 characters")
) )
func Connect(ctx context.Context, databaseURL string) (*Store, error) { func Connect(ctx context.Context, databaseURL string) (*Store, error) {
@@ -295,6 +302,7 @@ type PricingRule struct {
ResourceType string `json:"resourceType"` ResourceType string `json:"resourceType"`
Unit string `json:"unit"` Unit string `json:"unit"`
BasePrice float64 `json:"basePrice"` BasePrice float64 `json:"basePrice"`
IsFree bool `json:"isFree"`
Currency string `json:"currency"` Currency string `json:"currency"`
BaseWeight map[string]any `json:"baseWeight,omitempty"` BaseWeight map[string]any `json:"baseWeight,omitempty"`
DynamicWeight map[string]any `json:"dynamicWeight,omitempty"` DynamicWeight map[string]any `json:"dynamicWeight,omitempty"`
@@ -304,6 +312,8 @@ type PricingRule struct {
Priority int `json:"priority"` Priority int `json:"priority"`
Status string `json:"status"` Status string `json:"status"`
Metadata map[string]any `json:"metadata,omitempty"` Metadata map[string]any `json:"metadata,omitempty"`
EffectiveFrom string `json:"effectiveFrom,omitempty"`
EffectiveTo string `json:"effectiveTo,omitempty"`
CreatedAt time.Time `json:"createdAt"` CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"` UpdatedAt time.Time `json:"updatedAt"`
} }
@@ -411,64 +421,81 @@ type RateLimitWindow struct {
} }
type CreateTaskInput struct { type CreateTaskInput struct {
Kind string `json:"kind"` Kind string `json:"kind"`
Model string `json:"model"` Model string `json:"model"`
RunMode string `json:"runMode"` RunMode string `json:"runMode"`
Async bool `json:"async"` Async bool `json:"async"`
Request map[string]any `json:"request"` Request map[string]any `json:"request"`
ConversationID string `json:"conversationId"` ConversationID string `json:"conversationId"`
NewMessageCount int `json:"newMessageCount"` NewMessageCount int `json:"newMessageCount"`
MessageRefs []TaskMessageRefInput `json:"messageRefs"` MessageRefs []TaskMessageRefInput `json:"messageRefs"`
IdempotencyKeyHash string `json:"-"`
IdempotencyRequestHash string `json:"-"`
}
type CreateTaskResult struct {
Task GatewayTask
Replayed bool
} }
type GatewayTask struct { type GatewayTask struct {
ID string `json:"id"` ID string `json:"id"`
Kind string `json:"kind"` Kind string `json:"kind"`
RunMode string `json:"runMode"` RunMode string `json:"runMode"`
UserID string `json:"userId"` UserID string `json:"userId"`
GatewayUserID string `json:"gatewayUserId,omitempty"` GatewayUserID string `json:"gatewayUserId,omitempty"`
UserSource string `json:"userSource,omitempty"` UserSource string `json:"userSource,omitempty"`
GatewayTenantID string `json:"gatewayTenantId,omitempty"` GatewayTenantID string `json:"gatewayTenantId,omitempty"`
TenantID string `json:"tenantId,omitempty"` TenantID string `json:"tenantId,omitempty"`
TenantKey string `json:"tenantKey,omitempty"` TenantKey string `json:"tenantKey,omitempty"`
APIKeyID string `json:"apiKeyId,omitempty"` APIKeyID string `json:"apiKeyId,omitempty"`
APIKeyName string `json:"apiKeyName,omitempty"` APIKeyName string `json:"apiKeyName,omitempty"`
APIKeyPrefix string `json:"apiKeyPrefix,omitempty"` APIKeyPrefix string `json:"apiKeyPrefix,omitempty"`
UserGroupID string `json:"userGroupId,omitempty"` UserGroupID string `json:"userGroupId,omitempty"`
UserGroupKey string `json:"userGroupKey,omitempty"` UserGroupKey string `json:"userGroupKey,omitempty"`
Model string `json:"model"` Model string `json:"model"`
ModelType string `json:"modelType,omitempty"` ModelType string `json:"modelType,omitempty"`
RequestedModel string `json:"requestedModel,omitempty"` RequestedModel string `json:"requestedModel,omitempty"`
ResolvedModel string `json:"resolvedModel,omitempty"` ResolvedModel string `json:"resolvedModel,omitempty"`
RequestID string `json:"requestId,omitempty"` RequestID string `json:"requestId,omitempty"`
ConversationID string `json:"conversationId,omitempty"` ConversationID string `json:"conversationId,omitempty"`
NewMessageCount int `json:"newMessageCount,omitempty"` NewMessageCount int `json:"newMessageCount,omitempty"`
Request map[string]any `json:"request,omitempty"` Request map[string]any `json:"request,omitempty"`
AsyncMode bool `json:"asyncMode"` AsyncMode bool `json:"asyncMode"`
RiverJobID int64 `json:"riverJobId,omitempty"` RiverJobID int64 `json:"riverJobId,omitempty"`
Status string `json:"status"` Status string `json:"status"`
Cancellable *bool `json:"cancellable,omitempty"` Cancellable *bool `json:"cancellable,omitempty"`
Submitted *bool `json:"submitted,omitempty"` Submitted *bool `json:"submitted,omitempty"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
AttemptCount int `json:"attemptCount"` AttemptCount int `json:"attemptCount"`
RemoteTaskID string `json:"remoteTaskId,omitempty"` RemoteTaskID string `json:"remoteTaskId,omitempty"`
RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"` RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"`
Result map[string]any `json:"result,omitempty"` Result map[string]any `json:"result,omitempty"`
Billings []any `json:"billings,omitempty"` Billings []any `json:"billings,omitempty"`
Usage map[string]any `json:"usage"` Usage map[string]any `json:"usage"`
Metrics map[string]any `json:"metrics"` Metrics map[string]any `json:"metrics"`
BillingSummary map[string]any `json:"billingSummary"` BillingSummary map[string]any `json:"billingSummary"`
FinalChargeAmount float64 `json:"finalChargeAmount"` FinalChargeAmount float64 `json:"finalChargeAmount"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"` BillingVersion string `json:"billingVersion"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` BillingStatus string `json:"billingStatus"`
ResponseDurationMS int64 `json:"responseDurationMs"` BillingCurrency string `json:"billingCurrency"`
FinishedAt string `json:"finishedAt,omitempty"` PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
Error string `json:"error,omitempty"` RequestFingerprint string `json:"requestFingerprint,omitempty"`
ErrorCode string `json:"errorCode,omitempty"` ReservationAmount float64 `json:"reservationAmount"`
ErrorMessage string `json:"errorMessage,omitempty"` ExecutionToken string `json:"-"`
Attempts []TaskAttempt `json:"attempts,omitempty"` ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"`
CreatedAt time.Time `json:"createdAt"` BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"`
UpdatedAt time.Time `json:"updatedAt"` BillingSettledAt string `json:"billingSettledAt,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"`
FinishedAt string `json:"finishedAt,omitempty"`
Error string `json:"error,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
Attempts []TaskAttempt `json:"attempts,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
} }
const gatewayTaskColumns = ` const gatewayTaskColumns = `
@@ -482,7 +509,11 @@ request, COALESCE(async_mode, false), COALESCE(river_job_id, 0), status, COALESC
COALESCE(remote_task_id, ''), COALESCE(remote_task_payload, '{}'::jsonb), COALESCE(remote_task_id, ''), COALESCE(remote_task_payload, '{}'::jsonb),
COALESCE(result, '{}'::jsonb), COALESCE(billings, '[]'::jsonb), COALESCE(result, '{}'::jsonb), COALESCE(billings, '[]'::jsonb),
COALESCE(usage, '{}'::jsonb), COALESCE(metrics, '{}'::jsonb), COALESCE(billing_summary, '{}'::jsonb), COALESCE(usage, '{}'::jsonb), COALESCE(metrics, '{}'::jsonb), COALESCE(billing_summary, '{}'::jsonb),
COALESCE(final_charge_amount, 0)::float8, COALESCE(response_started_at::text, ''), COALESCE(final_charge_amount, 0)::float8, billing_version, billing_status, billing_currency,
COALESCE(pricing_snapshot, '{}'::jsonb), COALESCE(request_fingerprint, ''),
COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''),
COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''),
COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''),
COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''), COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''),
COALESCE(error_code, ''), COALESCE(error_message, ''), COALESCE(error_code, ''), COALESCE(error_message, ''),
created_at, updated_at, COALESCE(finished_at::text, '')` created_at, updated_at, COALESCE(finished_at::text, '')`
@@ -502,35 +533,39 @@ type TaskEvent struct {
} }
type TaskAttempt struct { type TaskAttempt struct {
ID string `json:"id"` ID string `json:"id"`
TaskID string `json:"taskId"` TaskID string `json:"taskId"`
AttemptNo int `json:"attemptNo"` AttemptNo int `json:"attemptNo"`
PlatformID string `json:"platformId,omitempty"` PlatformID string `json:"platformId,omitempty"`
PlatformName string `json:"platformName,omitempty"` PlatformName string `json:"platformName,omitempty"`
Provider string `json:"provider,omitempty"` Provider string `json:"provider,omitempty"`
PlatformModelID string `json:"platformModelId,omitempty"` PlatformModelID string `json:"platformModelId,omitempty"`
ModelName string `json:"modelName,omitempty"` ModelName string `json:"modelName,omitempty"`
ProviderModelName string `json:"providerModelName,omitempty"` ProviderModelName string `json:"providerModelName,omitempty"`
ModelAlias string `json:"modelAlias,omitempty"` ModelAlias string `json:"modelAlias,omitempty"`
ModelType string `json:"modelType,omitempty"` ModelType string `json:"modelType,omitempty"`
ClientID string `json:"clientId,omitempty"` ClientID string `json:"clientId,omitempty"`
QueueKey string `json:"queueKey"` QueueKey string `json:"queueKey"`
Status string `json:"status"` Status string `json:"status"`
Retryable bool `json:"retryable"` Retryable bool `json:"retryable"`
Simulated bool `json:"simulated"` Simulated bool `json:"simulated"`
RequestID string `json:"requestId,omitempty"` RequestID string `json:"requestId,omitempty"`
StatusCode int `json:"statusCode,omitempty"` StatusCode int `json:"statusCode,omitempty"`
Usage map[string]any `json:"usage,omitempty"` Usage map[string]any `json:"usage,omitempty"`
Metrics map[string]any `json:"metrics,omitempty"` Metrics map[string]any `json:"metrics,omitempty"`
RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"` RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"`
ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"` ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"`
ResponseStartedAt string `json:"responseStartedAt,omitempty"` ResponseStartedAt string `json:"responseStartedAt,omitempty"`
ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` ResponseFinishedAt string `json:"responseFinishedAt,omitempty"`
ResponseDurationMS int64 `json:"responseDurationMs"` ResponseDurationMS int64 `json:"responseDurationMs"`
ErrorCode string `json:"errorCode,omitempty"` PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"` RequestFingerprint string `json:"requestFingerprint,omitempty"`
StartedAt time.Time `json:"startedAt"` UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"`
FinishedAt string `json:"finishedAt,omitempty"` UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"`
ErrorCode string `json:"errorCode,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
StartedAt time.Time `json:"startedAt"`
FinishedAt string `json:"finishedAt,omitempty"`
} }
type TaskParamPreprocessingLog struct { type TaskParamPreprocessingLog struct {
@@ -1713,6 +1748,9 @@ ON CONFLICT (gateway_user_id, currency) DO NOTHING`,
); err != nil { ); err != nil {
return GatewayUser{}, err return GatewayUser{}, err
} }
if err := ensureWalletAccountAuditGuard(ctx, tx, user.ID, "resource"); err != nil {
return GatewayUser{}, err
}
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return GatewayUser{}, err return GatewayUser{}, err
} }
@@ -1823,6 +1861,11 @@ ORDER BY window_start DESC, scope_type ASC, scope_key ASC, metric ASC`)
} }
func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *auth.User) (GatewayTask, error) { func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *auth.User) (GatewayTask, error) {
result, err := s.CreateTaskIdempotent(ctx, input, user)
return result.Task, err
}
func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, user *auth.User) (CreateTaskResult, error) {
requestBody, _ := json.Marshal(input.Request) requestBody, _ := json.Marshal(input.Request)
runMode := normalizeRunMode(input.RunMode, input.Request) runMode := normalizeRunMode(input.RunMode, input.Request)
status := "queued" status := "queued"
@@ -1831,7 +1874,7 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
tx, err := s.pool.Begin(ctx) tx, err := s.pool.Begin(ctx)
if err != nil { if err != nil {
return GatewayTask{}, err return CreateTaskResult{}, err
} }
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
@@ -1839,33 +1882,62 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut
INSERT INTO gateway_tasks ( INSERT INTO gateway_tasks (
kind, run_mode, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key, kind, run_mode, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key,
api_key_id, api_key_name, api_key_prefix, user_group_id, user_group_key, api_key_id, api_key_name, api_key_prefix, user_group_id, user_group_key,
model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count, finished_at model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count,
idempotency_key_hash, idempotency_request_hash, finished_at
) )
VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, CASE WHEN $22 THEN now() ELSE NULL END) VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, NULLIF($22, ''), NULLIF($23, ''), NULL)
ON CONFLICT (user_id, idempotency_key_hash) WHERE idempotency_key_hash IS NOT NULL DO NOTHING
RETURNING `+gatewayTaskColumns, RETURNING `+gatewayTaskColumns,
input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, false, input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, strings.TrimSpace(input.IdempotencyKeyHash), strings.TrimSpace(input.IdempotencyRequestHash),
)) ))
replayed := false
if errors.Is(err, pgx.ErrNoRows) && strings.TrimSpace(input.IdempotencyKeyHash) != "" {
var existingRequestHash string
if err := tx.QueryRow(ctx, `
SELECT COALESCE(idempotency_request_hash, '')
FROM gateway_tasks
WHERE user_id = $1 AND idempotency_key_hash = $2
FOR UPDATE`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash)).Scan(&existingRequestHash); err != nil {
return CreateTaskResult{}, err
}
if existingRequestHash != strings.TrimSpace(input.IdempotencyRequestHash) {
return CreateTaskResult{}, ErrIdempotencyKeyReused
}
task, err = scanGatewayTask(tx.QueryRow(ctx, `
SELECT `+gatewayTaskColumns+`
FROM gateway_tasks
WHERE user_id = $1 AND idempotency_key_hash = $2`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash)))
replayed = true
}
if err != nil { if err != nil {
return GatewayTask{}, err return CreateTaskResult{}, err
} }
if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil { if !replayed {
return GatewayTask{}, err if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil {
} return CreateTaskResult{}, err
events := taskEventsForCreate(task.ID, runMode, status, nil) }
for _, event := range events { events := taskEventsForCreate(task.ID, runMode, status, nil)
payload, _ := json.Marshal(event.Payload) for _, event := range events {
if _, err := tx.Exec(ctx, ` payload, _ := json.Marshal(event.Payload)
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated) INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated)
VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`, VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`,
task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated, task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated,
); err != nil { ); err != nil {
return GatewayTask{}, err return CreateTaskResult{}, err
}
} }
} }
if err := tx.Commit(ctx); err != nil { if err := tx.Commit(ctx); err != nil {
return GatewayTask{}, err return CreateTaskResult{}, err
} }
return task, nil if replayed {
task, err = s.GetTask(ctx, task.ID)
if err != nil {
return CreateTaskResult{}, err
}
}
return CreateTaskResult{Task: task, Replayed: replayed}, nil
} }
func (s *Store) GetTask(ctx context.Context, taskID string) (GatewayTask, error) { func (s *Store) GetTask(ctx context.Context, taskID string) (GatewayTask, error) {
@@ -1897,6 +1969,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
var usageBytes []byte var usageBytes []byte
var metricsBytes []byte var metricsBytes []byte
var billingSummaryBytes []byte var billingSummaryBytes []byte
var pricingSnapshotBytes []byte
var remoteTaskPayloadBytes []byte var remoteTaskPayloadBytes []byte
if err := scanner.Scan( if err := scanner.Scan(
&task.ID, &task.ID,
@@ -1933,6 +2006,16 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
&metricsBytes, &metricsBytes,
&billingSummaryBytes, &billingSummaryBytes,
&task.FinalChargeAmount, &task.FinalChargeAmount,
&task.BillingVersion,
&task.BillingStatus,
&task.BillingCurrency,
&pricingSnapshotBytes,
&task.RequestFingerprint,
&task.ReservationAmount,
&task.ExecutionToken,
&task.ExecutionLeaseUntil,
&task.BillingUpdatedAt,
&task.BillingSettledAt,
&task.ResponseStartedAt, &task.ResponseStartedAt,
&task.ResponseFinishedAt, &task.ResponseFinishedAt,
&task.ResponseDurationMS, &task.ResponseDurationMS,
@@ -1952,6 +2035,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) {
task.Usage = decodeObject(usageBytes) task.Usage = decodeObject(usageBytes)
task.Metrics = decodeObject(metricsBytes) task.Metrics = decodeObject(metricsBytes)
task.BillingSummary = decodeObject(billingSummaryBytes) task.BillingSummary = decodeObject(billingSummaryBytes)
task.PricingSnapshot = decodeObject(pricingSnapshotBytes)
return task, nil return task, nil
} }
apps/api/internal/store/pricing_rules.go
+248 -9
View File
@@ -3,8 +3,10 @@ package store
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"fmt"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
) )
@@ -15,9 +17,10 @@ status, metadata, created_at, updated_at`
const pricingRuleColumns = ` const pricingRuleColumns = `
id::text, COALESCE(rule_set_id::text, ''), rule_key, display_name, scope_type, id::text, COALESCE(rule_set_id::text, ''), rule_key, display_name, scope_type,
COALESCE(scope_id::text, ''), resource_type, unit, base_price::float8, currency, COALESCE(scope_id::text, ''), resource_type, unit, base_price::float8, is_free, currency,
base_weight, dynamic_weight, calculator_type, dimension_schema, formula_config, base_weight, dynamic_weight, calculator_type, dimension_schema, formula_config,
priority, status, metadata, created_at, updated_at` priority, status, metadata, COALESCE(effective_from::text, ''), COALESCE(effective_to::text, ''),
created_at, updated_at`
type PricingRuleInput struct { type PricingRuleInput struct {
RuleKey string `json:"ruleKey"` RuleKey string `json:"ruleKey"`
@@ -25,6 +28,7 @@ type PricingRuleInput struct {
ResourceType string `json:"resourceType"` ResourceType string `json:"resourceType"`
Unit string `json:"unit"` Unit string `json:"unit"`
BasePrice float64 `json:"basePrice"` BasePrice float64 `json:"basePrice"`
IsFree bool `json:"isFree"`
Currency string `json:"currency"` Currency string `json:"currency"`
BaseWeight map[string]any `json:"baseWeight"` BaseWeight map[string]any `json:"baseWeight"`
DynamicWeight map[string]any `json:"dynamicWeight"` DynamicWeight map[string]any `json:"dynamicWeight"`
@@ -34,6 +38,17 @@ type PricingRuleInput struct {
Priority int `json:"priority"` Priority int `json:"priority"`
Status string `json:"status"` Status string `json:"status"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
EffectiveFrom string `json:"effectiveFrom"`
EffectiveTo string `json:"effectiveTo"`
}
type EffectivePricingConfig struct {
RuleSetID string
RuleSetKey string
Currency string
Config map[string]any
FreeResource map[string]bool
Snapshot map[string]any
} }
type PricingRuleSetInput struct { type PricingRuleSetInput struct {
@@ -247,6 +262,221 @@ ORDER BY priority ASC, resource_type ASC`, id)
return config, nil return config, nil
} }
func (s *Store) PricingRuleSetBillingConfigV2(ctx context.Context, id string) (EffectivePricingConfig, error) {
id = strings.TrimSpace(id)
if id == "" {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule set id is required")
}
var ruleSetKey string
var currency string
var status string
if err := s.pool.QueryRow(ctx, `
SELECT rule_set_key, currency, status
FROM model_pricing_rule_sets
WHERE id = $1::uuid`, id).Scan(&ruleSetKey, &currency, &status); err != nil {
return EffectivePricingConfig{}, err
}
if status != "active" {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s is not active", id)
}
if currency != "resource" {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s uses unsupported currency %s", id, currency)
}
rows, err := s.pool.Query(ctx, `
SELECT rule_key, resource_type, unit, base_price::text, currency, base_weight,
dynamic_weight, calculator_type, dimension_schema, formula_config,
priority, is_free, COALESCE(effective_from::text, ''), COALESCE(effective_to::text, '')
FROM model_pricing_rules
WHERE rule_set_id = $1::uuid
AND status = 'active'
AND (effective_from IS NULL OR effective_from <= now())
AND (effective_to IS NULL OR effective_to > now())
ORDER BY priority ASC, resource_type ASC, rule_key ASC`, id)
if err != nil {
return EffectivePricingConfig{}, err
}
defer rows.Close()
config := map[string]any{}
freeResource := map[string]bool{}
rules := make([]any, 0)
seen := map[string]bool{}
for rows.Next() {
var ruleKey string
var resourceType string
var unit string
var basePrice string
var ruleCurrency string
var baseWeightBytes []byte
var dynamicWeightBytes []byte
var calculatorType string
var dimensionSchemaBytes []byte
var formulaConfigBytes []byte
var priority int
var isFree bool
var effectiveFrom string
var effectiveTo string
if err := rows.Scan(
&ruleKey, &resourceType, &unit, &basePrice, &ruleCurrency, &baseWeightBytes,
&dynamicWeightBytes, &calculatorType, &dimensionSchemaBytes, &formulaConfigBytes,
&priority, &isFree, &effectiveFrom, &effectiveTo,
); err != nil {
return EffectivePricingConfig{}, err
}
if seen[resourceType] {
continue
}
if ruleCurrency != currency {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s currency does not match its rule set", ruleKey)
}
switch calculatorType {
case "token_usage", "unit_weight", "duration_weight":
default:
return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s uses unsupported calculator %s", ruleKey, calculatorType)
}
if err := ValidateEffectivePricingRuleShape(resourceType, unit, calculatorType); err != nil {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s: %w", ruleKey, err)
}
if strings.HasPrefix(strings.TrimSpace(basePrice), "-") {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s has a negative base price", ruleKey)
}
dynamicWeight := decodeObject(dynamicWeightBytes)
formulaConfig := decodeObject(formulaConfigBytes)
addPricingRuleToConfig(config, resourceType, basePrice, dynamicWeight, formulaConfig)
freeResource[resourceType] = isFree
seen[resourceType] = true
rules = append(rules, map[string]any{
"ruleKey": ruleKey, "resourceType": resourceType, "unit": unit,
"basePrice": basePrice, "currency": ruleCurrency, "baseWeight": decodeObject(baseWeightBytes),
"dynamicWeight": dynamicWeight, "calculatorType": calculatorType,
"dimensionSchema": decodeObject(dimensionSchemaBytes), "formulaConfig": formulaConfig,
"priority": priority, "isFree": isFree,
"effectiveFrom": effectiveFrom, "effectiveTo": effectiveTo,
})
}
if err := rows.Err(); err != nil {
return EffectivePricingConfig{}, err
}
if len(rules) == 0 {
return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s has no effective rules", id)
}
return EffectivePricingConfig{
RuleSetID: id, RuleSetKey: ruleSetKey, Currency: currency, Config: config,
FreeResource: freeResource,
Snapshot: map[string]any{
"pricingVersion": "effective-pricing-v2",
"ruleSetId": id,
"ruleSetKey": ruleSetKey,
"currency": currency,
"resolvedAt": time.Now().UTC().Format(time.RFC3339Nano),
"rules": rules,
},
}, nil
}
func ValidateEffectivePricingRuleShape(resourceType string, unit string, calculatorType string) error {
unit = NormalizeEffectivePricingRuleUnit(resourceType, unit)
if strings.TrimSpace(calculatorType) == "" {
calculatorType = DefaultEffectivePricingCalculator(resourceType)
}
allowed := false
switch resourceType {
case "text_input", "text_cached_input", "text_output", "text_total":
allowed = unit == "1k_tokens" && calculatorType == "token_usage"
case "image", "image_edit":
allowed = unit == "image" && calculatorType == "unit_weight"
case "video":
allowed = unit == "5s" && calculatorType == "duration_weight"
case "music":
allowed = (unit == "song" || unit == "item") && calculatorType == "unit_weight"
case "audio":
allowed = unit == "character" && calculatorType == "unit_weight"
default:
return nil
}
if !allowed {
return fmt.Errorf("unit %q and calculator %q do not match resource %q", unit, calculatorType, resourceType)
}
return nil
}
func NormalizeEffectivePricingRuleUnit(resourceType string, unit string) string {
unit = strings.TrimSpace(unit)
if resourceType == "video" && unit == "video" {
return "5s"
}
return unit
}
func DefaultEffectivePricingCalculator(resourceType string) string {
switch strings.TrimSpace(resourceType) {
case "text_input", "text_cached_input", "text_output", "text_total":
return "token_usage"
case "video":
return "duration_weight"
default:
return "unit_weight"
}
}
func addPricingRuleToConfig(config map[string]any, resourceType string, basePrice string, dynamicWeight map[string]any, formulaConfig map[string]any) {
resourceConfig := map[string]any{"basePrice": basePrice}
if len(dynamicWeight) > 0 {
resourceConfig["dynamicWeight"] = dynamicWeight
}
if len(formulaConfig) > 0 {
resourceConfig["formulaConfig"] = formulaConfig
}
switch resourceType {
case "text_input":
config["textInputPer1k"] = basePrice
case "text_cached_input":
config["textCachedInputPer1k"] = basePrice
case "text_output":
config["textOutputPer1k"] = basePrice
case "text_total":
inputPrice := any(basePrice)
if value, ok := pricingRuleValueFromKeys(formulaConfig, "inputTokenPrice", "input_token_price", "textInputPer1k", "text_input"); ok {
inputPrice = value
}
config["textInputPer1k"] = inputPrice
if value, ok := pricingRuleValueFromKeys(formulaConfig, "cachedInputTokenPrice", "cached_input_token_price", "textCachedInputPer1k", "text_cached_input", "inputCacheHitTokenPrice", "input_cache_hit_token_price"); ok {
config["textCachedInputPer1k"] = value
}
if value, ok := pricingRuleValueFromKeys(formulaConfig, "outputTokenPrice", "output_token_price", "textOutputPer1k", "text_output"); ok {
config["textOutputPer1k"] = value
}
config["text_total"] = resourceConfig
case "image":
config["imageBase"] = basePrice
config["image"] = resourceConfig
case "image_edit":
config["editBase"] = basePrice
config["image_edit"] = resourceConfig
case "video":
config["videoBase"] = basePrice
config["video"] = resourceConfig
case "music":
config["musicBase"] = basePrice
config["music"] = resourceConfig
case "audio":
config["audioBase"] = basePrice
config["audio"] = resourceConfig
default:
config[resourceType] = resourceConfig
}
}
func pricingRuleValueFromKeys(config map[string]any, keys ...string) (any, bool) {
for _, key := range keys {
if value, ok := config[key]; ok {
return value, true
}
}
return nil, false
}
func pricingRuleNumberFromKeys(config map[string]any, keys ...string) (float64, bool) { func pricingRuleNumberFromKeys(config map[string]any, keys ...string) (float64, bool) {
if len(config) == 0 { if len(config) == 0 {
return 0, false return 0, false
@@ -305,13 +535,16 @@ func insertPricingRules(ctx context.Context, tx pgx.Tx, ruleSetID string, defaul
if _, err := tx.Exec(ctx, ` if _, err := tx.Exec(ctx, `
INSERT INTO model_pricing_rules ( INSERT INTO model_pricing_rules (
rule_set_id, rule_key, display_name, scope_type, scope_id, resource_type, rule_set_id, rule_key, display_name, scope_type, scope_id, resource_type,
unit, base_price, currency, base_weight, dynamic_weight, calculator_type, unit, base_price, is_free, currency, base_weight, dynamic_weight, calculator_type,
dimension_schema, formula_config, priority, status, metadata dimension_schema, formula_config, priority, status, metadata, effective_from, effective_to
) )
VALUES ($1::uuid, $2, $3, 'rule_set', $1::uuid, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)`, VALUES (
$1::uuid, $2, $3, 'rule_set', $1::uuid, $4, $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, NULLIF($17, '')::timestamptz, NULLIF($18, '')::timestamptz
)`,
ruleSetID, rule.RuleKey, rule.DisplayName, rule.ResourceType, rule.Unit, ruleSetID, rule.RuleKey, rule.DisplayName, rule.ResourceType, rule.Unit,
rule.BasePrice, rule.Currency, baseWeight, dynamicWeight, rule.CalculatorType, rule.BasePrice, rule.IsFree, rule.Currency, baseWeight, dynamicWeight, rule.CalculatorType,
dimensionSchema, formulaConfig, rule.Priority, rule.Status, metadata, dimensionSchema, formulaConfig, rule.Priority, rule.Status, metadata, rule.EffectiveFrom, rule.EffectiveTo,
); err != nil { ); err != nil {
return err return err
} }
@@ -358,6 +591,7 @@ func scanPricingRule(scanner pricingScanner) (PricingRule, error) {
&item.ResourceType, &item.ResourceType,
&item.Unit, &item.Unit,
&item.BasePrice, &item.BasePrice,
&item.IsFree,
&item.Currency, &item.Currency,
&baseWeight, &baseWeight,
&dynamicWeight, &dynamicWeight,
@@ -367,6 +601,8 @@ func scanPricingRule(scanner pricingScanner) (PricingRule, error) {
&item.Priority, &item.Priority,
&item.Status, &item.Status,
&metadata, &metadata,
&item.EffectiveFrom,
&item.EffectiveTo,
&item.CreatedAt, &item.CreatedAt,
&item.UpdatedAt, &item.UpdatedAt,
); err != nil { ); err != nil {
@@ -403,7 +639,7 @@ func normalizePricingRule(input PricingRuleInput, index int, defaultCurrency str
input.RuleKey = strings.TrimSpace(input.RuleKey) input.RuleKey = strings.TrimSpace(input.RuleKey)
input.DisplayName = strings.TrimSpace(input.DisplayName) input.DisplayName = strings.TrimSpace(input.DisplayName)
input.ResourceType = strings.TrimSpace(input.ResourceType) input.ResourceType = strings.TrimSpace(input.ResourceType)
input.Unit = strings.TrimSpace(input.Unit) input.Unit = NormalizeEffectivePricingRuleUnit(input.ResourceType, input.Unit)
input.Currency = strings.TrimSpace(input.Currency) input.Currency = strings.TrimSpace(input.Currency)
input.CalculatorType = strings.TrimSpace(input.CalculatorType) input.CalculatorType = strings.TrimSpace(input.CalculatorType)
input.Status = strings.TrimSpace(input.Status) input.Status = strings.TrimSpace(input.Status)
@@ -420,7 +656,7 @@ func normalizePricingRule(input PricingRuleInput, index int, defaultCurrency str
input.Currency = defaultCurrency input.Currency = defaultCurrency
} }
if input.CalculatorType == "" { if input.CalculatorType == "" {
input.CalculatorType = "unit_weight" input.CalculatorType = DefaultEffectivePricingCalculator(input.ResourceType)
} }
if input.Priority == 0 { if input.Priority == 0 {
input.Priority = (index + 1) * 10 input.Priority = (index + 1) * 10
@@ -444,6 +680,7 @@ func pricingInputsToRules(ruleSetID string, defaultCurrency string, rules []Pric
ResourceType: input.ResourceType, ResourceType: input.ResourceType,
Unit: input.Unit, Unit: input.Unit,
BasePrice: input.BasePrice, BasePrice: input.BasePrice,
IsFree: input.IsFree,
Currency: input.Currency, Currency: input.Currency,
BaseWeight: emptyObjectIfNil(input.BaseWeight), BaseWeight: emptyObjectIfNil(input.BaseWeight),
DynamicWeight: emptyObjectIfNil(input.DynamicWeight), DynamicWeight: emptyObjectIfNil(input.DynamicWeight),
@@ -453,6 +690,8 @@ func pricingInputsToRules(ruleSetID string, defaultCurrency string, rules []Pric
Priority: input.Priority, Priority: input.Priority,
Status: input.Status, Status: input.Status,
Metadata: emptyObjectIfNil(input.Metadata), Metadata: emptyObjectIfNil(input.Metadata),
EffectiveFrom: input.EffectiveFrom,
EffectiveTo: input.EffectiveTo,
}) })
} }
return items return items
apps/api/internal/store/pricing_rules_test.go
+27
View File
@@ -0,0 +1,27 @@
package store
import "testing"
func TestValidateEffectivePricingRuleShape(t *testing.T) {
tests := []struct {
name string
resource string
unit string
calculator string
valid bool
}{
{name: "text", resource: "text_input", unit: "1k_tokens", calculator: "token_usage", valid: true},
{name: "image", resource: "image", unit: "image", calculator: "unit_weight", valid: true},
{name: "video", resource: "video", unit: "5s", calculator: "duration_weight", valid: true},
{name: "wrong video unit", resource: "video", unit: "second", calculator: "duration_weight"},
{name: "wrong image calculator", resource: "image", unit: "image", calculator: "token_usage"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
err := ValidateEffectivePricingRuleShape(test.resource, test.unit, test.calculator)
if (err == nil) != test.valid {
t.Fatalf("valid=%v err=%v", test.valid, err)
}
})
}
}
apps/api/internal/store/runtime_types.go
+44 -18
View File
@@ -223,21 +223,24 @@ type RateLimitResult struct {
} }
type CreateTaskAttemptInput struct { type CreateTaskAttemptInput struct {
TaskID string TaskID string
AttemptNo int AttemptNo int
PlatformID string PlatformID string
PlatformModelID string PlatformModelID string
ClientID string ClientID string
QueueKey string QueueKey string
Status string Status string
Simulated bool Simulated bool
RequestSnapshot map[string]any RequestSnapshot map[string]any
Metrics map[string]any Metrics map[string]any
PricingSnapshot map[string]any
RequestFingerprint string
} }
type AsyncTaskQueueItem struct { type AsyncTaskQueueItem struct {
ID string ID string
Priority int Priority int
NextRunAt time.Time
} }
type FinishTaskAttemptInput struct { type FinishTaskAttemptInput struct {
@@ -256,15 +259,37 @@ type FinishTaskAttemptInput struct {
} }
type FinishTaskSuccessInput struct { type FinishTaskSuccessInput struct {
TaskID string
ExecutionToken string
AttemptID string
Result map[string]any
Billings []any
RequestID string
ResolvedModel string
Usage map[string]any
Metrics map[string]any
BillingSummary map[string]any
FinalChargeAmount float64
FinalChargeAmountText string
BillingCurrency string
PricingSnapshot map[string]any
RequestFingerprint string
ResponseStartedAt time.Time
ResponseFinishedAt time.Time
ResponseDurationMS int64
}
type FinishTaskManualReviewInput struct {
TaskID string TaskID string
ExecutionToken string
AttemptID string
TaskStatus string
Code string
Message string
Result map[string]any Result map[string]any
Billings []any
RequestID string RequestID string
ResolvedModel string PricingSnapshot map[string]any
Usage map[string]any RequestFingerprint string
Metrics map[string]any
BillingSummary map[string]any
FinalChargeAmount float64
ResponseStartedAt time.Time ResponseStartedAt time.Time
ResponseFinishedAt time.Time ResponseFinishedAt time.Time
ResponseDurationMS int64 ResponseDurationMS int64
@@ -272,6 +297,7 @@ type FinishTaskSuccessInput struct {
type FinishTaskFailureInput struct { type FinishTaskFailureInput struct {
TaskID string TaskID string
ExecutionToken string
Code string Code string
Message string Message string
Result map[string]any Result map[string]any
apps/api/internal/store/tasks_runtime.go
+436 -54
View File
@@ -3,6 +3,7 @@ package store
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
@@ -152,18 +153,163 @@ func nullableTaskListTime(value *time.Time) any {
return *value return *value
} }
func (s *Store) MarkTaskRunning(ctx context.Context, taskID string, modelType string, normalizedRequest map[string]any) error { func (s *Store) ClaimTaskExecution(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) (GatewayTask, error) {
normalizedJSON, _ := json.Marshal(emptyObjectIfNil(normalizedRequest)) if leaseTTL <= 0 {
_, err := s.pool.Exec(ctx, ` leaseTTL = 5 * time.Minute
}
var task GatewayTask
manualReview := false
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var queuedReady bool
var runningExpired bool
var production bool
var hasGatewayUser bool
if err := tx.QueryRow(ctx, `
SELECT status = 'queued' AND next_run_at <= now(),
status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()),
run_mode = 'production',
gateway_user_id IS NOT NULL
FROM gateway_tasks
WHERE id = $1::uuid
FOR UPDATE`, taskID).Scan(&queuedReady, &runningExpired, &production, &hasGatewayUser); err != nil {
return err
}
if !queuedReady && !runningExpired {
return ErrTaskExecutionLeaseUnavailable
}
if runningExpired && production {
var submissionAmbiguous bool
if err := tx.QueryRow(ctx, `
SELECT COALESCE((
SELECT (
(status = 'running' AND upstream_submission_status IN ('submitting', 'response_received'))
OR (status = 'failed' AND upstream_submission_status = 'submitting')
)
FROM gateway_task_attempts
WHERE task_id = $1::uuid
ORDER BY attempt_no DESC, started_at DESC
LIMIT 1
), false)`, taskID).Scan(&submissionAmbiguous); err != nil {
return err
}
if submissionAmbiguous {
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'failed',
billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END,
billing_updated_at = now(),
error = 'upstream submission result is unknown',
error_code = 'upstream_submission_unknown',
error_message = 'upstream submission result is unknown',
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil {
return err
}
if hasGatewayUser {
payloadJSON, _ := json.Marshal(map[string]any{
"taskId": taskID, "classification": "upstream_submission_unknown",
})
if _, err := tx.Exec(ctx, `
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
status, next_attempt_at, manual_review_reason
)
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown'
FROM gateway_tasks
WHERE id = $1::uuid
ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil {
return err
}
}
manualReview = true
return nil
}
}
var err error
task, err = scanGatewayTask(tx.QueryRow(ctx, `
UPDATE gateway_tasks UPDATE gateway_tasks
SET status = 'running', SET status = 'running',
model_type = NULLIF($2::text, ''), execution_token = $2::uuid,
normalized_request = $3::jsonb, execution_lease_expires_at = now() + ($3::int * interval '1 second'),
locked_at = now(), locked_at = now(),
heartbeat_at = now(), heartbeat_at = now(),
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, taskID, modelType, string(normalizedJSON)) WHERE id = $1::uuid
return err AND (
(status = 'queued' AND next_run_at <= now())
OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()))
)
RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second)))
return err
})
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return GatewayTask{}, ErrTaskExecutionLeaseUnavailable
}
return GatewayTask{}, err
}
if manualReview {
return GatewayTask{}, ErrTaskExecutionManualReview
}
return task, nil
}
func (s *Store) RenewTaskExecutionLease(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) error {
if leaseTTL <= 0 {
leaseTTL = 5 * time.Minute
}
tag, err := s.pool.Exec(ctx, `
UPDATE gateway_tasks
SET execution_lease_expires_at = now() + ($3::int * interval '1 second'),
heartbeat_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $2::uuid`, taskID, executionToken, int(leaseTTL/time.Second))
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
var stillRunning bool
if err := s.pool.QueryRow(ctx, `
SELECT COALESCE((SELECT status = 'running' FROM gateway_tasks WHERE id = $1::uuid), false)`, taskID).Scan(&stillRunning); err != nil {
return err
}
if !stillRunning {
return ErrTaskExecutionFinished
}
return ErrTaskExecutionLeaseLost
}
return nil
}
func (s *Store) MarkTaskRunning(ctx context.Context, taskID string, executionToken string, modelType string, normalizedRequest map[string]any) error {
normalizedJSON, _ := json.Marshal(emptyObjectIfNil(normalizedRequest))
tag, err := s.pool.Exec(ctx, `
UPDATE gateway_tasks
SET status = 'running',
model_type = NULLIF($3::text, ''),
normalized_request = $4::jsonb,
locked_at = now(),
heartbeat_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $2::uuid`, taskID, executionToken, modelType, string(normalizedJSON))
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return ErrTaskExecutionLeaseLost
}
return nil
} }
func (s *Store) ClaimAsyncQueuedTask(ctx context.Context, workerID string) (GatewayTask, error) { func (s *Store) ClaimAsyncQueuedTask(ctx context.Context, workerID string) (GatewayTask, error) {
@@ -189,7 +335,7 @@ WHERE t.id = picked.task_id
RETURNING `+gatewayTaskColumns, workerID)) RETURNING `+gatewayTaskColumns, workerID))
} }
func (s *Store) RequeueTask(ctx context.Context, taskID string, delay time.Duration, queueKey string) (GatewayTask, error) { func (s *Store) RequeueTask(ctx context.Context, taskID string, executionToken string, delay time.Duration, queueKey string) (GatewayTask, error) {
if delay < time.Second { if delay < time.Second {
delay = time.Second delay = time.Second
} }
@@ -203,14 +349,18 @@ SET status = 'queued',
locked_by = NULL, locked_by = NULL,
locked_at = NULL, locked_at = NULL,
heartbeat_at = NULL, heartbeat_at = NULL,
next_run_at = $2::timestamptz, execution_token = NULL,
queue_key = COALESCE(NULLIF($3::text, ''), queue_key), execution_lease_expires_at = NULL,
next_run_at = $3::timestamptz,
queue_key = COALESCE(NULLIF($4::text, ''), queue_key),
error = NULL, error = NULL,
error_code = NULL, error_code = NULL,
error_message = NULL, error_message = NULL,
updated_at = now() updated_at = now()
WHERE id = $1::uuid WHERE id = $1::uuid
RETURNING `+gatewayTaskColumns, taskID, nextRunAt, strings.TrimSpace(queueKey))) AND status = 'running'
AND execution_token = $2::uuid
RETURNING `+gatewayTaskColumns, taskID, executionToken, nextRunAt, strings.TrimSpace(queueKey)))
} }
func (s *Store) SetTaskRiverJobID(ctx context.Context, taskID string, riverJobID int64) error { func (s *Store) SetTaskRiverJobID(ctx context.Context, taskID string, riverJobID int64) error {
@@ -225,25 +375,32 @@ WHERE id = $1::uuid`, taskID, riverJobID)
return err return err
} }
func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, attemptID string, remoteTaskID string, payload map[string]any) error { func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error {
payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload)) payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload))
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, ` tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks UPDATE gateway_tasks
SET remote_task_id = NULLIF($2::text, ''), SET remote_task_id = NULLIF($3::text, ''),
remote_task_payload = $3::jsonb, remote_task_payload = $4::jsonb,
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $2::uuid`,
taskID, taskID,
executionToken,
remoteTaskID, remoteTaskID,
string(payloadJSON), string(payloadJSON),
); err != nil { )
if err != nil {
return err return err
} }
if tag.RowsAffected() != 1 {
return ErrTaskExecutionLeaseLost
}
if strings.TrimSpace(attemptID) == "" { if strings.TrimSpace(attemptID) == "" {
return nil return nil
} }
_, err := tx.Exec(ctx, ` _, err = tx.Exec(ctx, `
UPDATE gateway_task_attempts UPDATE gateway_task_attempts
SET remote_task_id = NULLIF($2::text, ''), SET remote_task_id = NULLIF($2::text, ''),
response_snapshot = COALESCE(response_snapshot, '{}'::jsonb) || jsonb_build_object('remote_task_payload', $3::jsonb) response_snapshot = COALESCE(response_snapshot, '{}'::jsonb) || jsonb_build_object('remote_task_payload', $3::jsonb)
@@ -256,6 +413,20 @@ WHERE id = $1::uuid`,
}) })
} }
func (s *Store) SetAttemptUpstreamSubmissionStatus(ctx context.Context, attemptID string, status string) error {
switch status {
case "not_submitted", "submitting", "response_received":
default:
return fmt.Errorf("invalid upstream submission status %q", status)
}
_, err := s.pool.Exec(ctx, `
UPDATE gateway_task_attempts
SET upstream_submission_status = $2,
upstream_submission_updated_at = now()
WHERE id = $1::uuid`, attemptID, status)
return err
}
func (s *Store) CancelQueuedTask(ctx context.Context, taskID string, message string) (GatewayTask, bool, error) { func (s *Store) CancelQueuedTask(ctx context.Context, taskID string, message string) (GatewayTask, bool, error) {
message = strings.TrimSpace(message) message = strings.TrimSpace(message)
if message == "" { if message == "" {
@@ -293,10 +464,13 @@ func (s *Store) ListRecoverableAsyncTasks(ctx context.Context, limit int) ([]Asy
limit = 500 limit = 500
} }
rows, err := s.pool.Query(ctx, ` rows, err := s.pool.Query(ctx, `
SELECT id::text, priority SELECT id::text, priority, next_run_at
FROM gateway_tasks FROM gateway_tasks
WHERE async_mode = true WHERE async_mode = true
AND status IN ('queued', 'running') AND (
status = 'queued'
OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()))
)
ORDER BY priority ASC, created_at ASC ORDER BY priority ASC, created_at ASC
LIMIT $1`, limit) LIMIT $1`, limit)
if err != nil { if err != nil {
@@ -306,7 +480,7 @@ LIMIT $1`, limit)
items := make([]AsyncTaskQueueItem, 0) items := make([]AsyncTaskQueueItem, 0)
for rows.Next() { for rows.Next() {
var item AsyncTaskQueueItem var item AsyncTaskQueueItem
if err := rows.Scan(&item.ID, &item.Priority); err != nil { if err := rows.Scan(&item.ID, &item.Priority, &item.NextRunAt); err != nil {
return nil, err return nil, err
} }
items = append(items, item) items = append(items, item)
@@ -320,6 +494,7 @@ LIMIT $1`, limit)
func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptInput) (string, error) { func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptInput) (string, error) {
requestJSON, _ := json.Marshal(emptyObjectIfNil(input.RequestSnapshot)) requestJSON, _ := json.Marshal(emptyObjectIfNil(input.RequestSnapshot))
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
pricingJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
tx, err := s.pool.Begin(ctx) tx, err := s.pool.Begin(ctx)
if err != nil { if err != nil {
return "", err return "", err
@@ -330,11 +505,13 @@ func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptIn
err = tx.QueryRow(ctx, ` err = tx.QueryRow(ctx, `
INSERT INTO gateway_task_attempts ( INSERT INTO gateway_task_attempts (
task_id, attempt_no, platform_id, platform_model_id, client_id, queue_key, task_id, attempt_no, platform_id, platform_model_id, client_id, queue_key,
status, simulated, request_snapshot, metrics status, simulated, request_snapshot, metrics, pricing_snapshot, request_fingerprint,
upstream_submission_status, upstream_submission_updated_at
) )
VALUES ( VALUES (
$1::uuid, $2::int, NULLIF($3::text, '')::uuid, NULLIF($4::text, '')::uuid, NULLIF($5::text, ''), $6, $1::uuid, $2::int, NULLIF($3::text, '')::uuid, NULLIF($4::text, '')::uuid, NULLIF($5::text, ''), $6,
$7, $8, $9::jsonb, $10::jsonb $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, NULLIF($12, ''),
'not_submitted', now()
) )
RETURNING id::text`, RETURNING id::text`,
input.TaskID, input.TaskID,
@@ -347,6 +524,8 @@ RETURNING id::text`,
input.Simulated, input.Simulated,
string(requestJSON), string(requestJSON),
string(metricsJSON), string(metricsJSON),
string(pricingJSON),
input.RequestFingerprint,
).Scan(&attemptID) ).Scan(&attemptID)
if err != nil { if err != nil {
return "", err return "", err
@@ -489,6 +668,8 @@ SELECT a.id::text, a.task_id::text, a.attempt_no,
a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb), a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb),
COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''), COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''),
COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''), COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''),
COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''),
a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''),
a.started_at, COALESCE(a.finished_at::text, '') a.started_at, COALESCE(a.finished_at::text, '')
FROM gateway_task_attempts a FROM gateway_task_attempts a
LEFT JOIN integration_platforms p ON p.id = a.platform_id LEFT JOIN integration_platforms p ON p.id = a.platform_id
@@ -518,6 +699,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
var metricsBytes []byte var metricsBytes []byte
var requestBytes []byte var requestBytes []byte
var responseBytes []byte var responseBytes []byte
var pricingSnapshotBytes []byte
if err := scanner.Scan( if err := scanner.Scan(
&item.ID, &item.ID,
&item.TaskID, &item.TaskID,
@@ -544,6 +726,10 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
&item.ResponseDurationMS, &item.ResponseDurationMS,
&item.ErrorCode, &item.ErrorCode,
&item.ErrorMessage, &item.ErrorMessage,
&pricingSnapshotBytes,
&item.RequestFingerprint,
&item.UpstreamSubmissionStatus,
&item.UpstreamSubmissionUpdatedAt,
&item.StartedAt, &item.StartedAt,
&item.FinishedAt, &item.FinishedAt,
); err != nil { ); err != nil {
@@ -553,6 +739,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) {
item.Metrics = decodeObject(metricsBytes) item.Metrics = decodeObject(metricsBytes)
item.RequestSnapshot = decodeObject(requestBytes) item.RequestSnapshot = decodeObject(requestBytes)
item.ResponseSnapshot = decodeObject(responseBytes) item.ResponseSnapshot = decodeObject(responseBytes)
item.PricingSnapshot = decodeObject(pricingSnapshotBytes)
enrichTaskAttemptFromMetrics(&item) enrichTaskAttemptFromMetrics(&item)
return item, nil return item, nil
} }
@@ -670,7 +857,41 @@ func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessIn
usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage)) usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage))
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary)) billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary))
if _, err := s.pool.Exec(ctx, ` pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText)
if finalChargeAmount == "" {
finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64)
}
currency := normalizeWalletCurrency(input.BillingCurrency)
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if strings.TrimSpace(input.AttemptID) != "" {
if _, err := tx.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = 'succeeded',
retryable = false,
request_id = NULLIF($2, ''),
usage = $3::jsonb,
metrics = $4::jsonb,
response_snapshot = $5::jsonb,
pricing_snapshot = $6::jsonb,
request_fingerprint = NULLIF($7, ''),
upstream_submission_status = 'response_received',
upstream_submission_updated_at = now(),
response_started_at = $8::timestamptz,
response_finished_at = $9::timestamptz,
response_duration_ms = $10,
error_code = NULL,
error_message = NULL,
finished_at = now()
WHERE id = $1::uuid`,
input.AttemptID, input.RequestID, string(usageJSON), string(metricsJSON),
string(resultJSON), string(pricingSnapshotJSON), input.RequestFingerprint,
nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS,
); err != nil {
return err
}
}
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks UPDATE gateway_tasks
SET status = 'succeeded', SET status = 'succeeded',
result = $2::jsonb, result = $2::jsonb,
@@ -680,32 +901,156 @@ SET status = 'succeeded',
usage = $6::jsonb, usage = $6::jsonb,
metrics = $7::jsonb, metrics = $7::jsonb,
billing_summary = $8::jsonb, billing_summary = $8::jsonb,
final_charge_amount = $9, final_charge_amount = $9::numeric,
response_started_at = $10::timestamptz, billing_version = 'effective-pricing-v2',
response_finished_at = $11::timestamptz, billing_status = CASE
response_duration_ms = $12, WHEN run_mode = 'production' AND gateway_user_id IS NOT NULL THEN 'pending'
ELSE 'not_required'
END,
billing_currency = $10,
pricing_snapshot = $11::jsonb,
request_fingerprint = NULLIF($12, ''),
billing_updated_at = now(),
response_started_at = $13::timestamptz,
response_finished_at = $14::timestamptz,
response_duration_ms = $15,
error = NULL, error = NULL,
error_code = NULL, error_code = NULL,
error_message = NULL, error_message = NULL,
locked_by = NULL, locked_by = NULL,
locked_at = NULL, locked_at = NULL,
heartbeat_at = NULL, heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(), finished_at = now(),
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, WHERE id = $1::uuid
input.TaskID, AND status = 'running'
string(resultJSON), AND execution_token = $16::uuid`,
string(billingsJSON), input.TaskID,
input.RequestID, string(resultJSON),
input.ResolvedModel, string(billingsJSON),
string(usageJSON), input.RequestID,
string(metricsJSON), input.ResolvedModel,
string(billingSummaryJSON), string(usageJSON),
input.FinalChargeAmount, string(metricsJSON),
nullableTime(input.ResponseStartedAt), string(billingSummaryJSON),
nullableTime(input.ResponseFinishedAt), finalChargeAmount,
input.ResponseDurationMS, currency,
); err != nil { string(pricingSnapshotJSON),
input.RequestFingerprint,
nullableTime(input.ResponseStartedAt),
nullableTime(input.ResponseFinishedAt),
input.ResponseDurationMS,
input.ExecutionToken,
)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return ErrTaskExecutionLeaseLost
}
payloadJSON, _ := json.Marshal(map[string]any{"taskId": input.TaskID, "pricingVersion": "effective-pricing-v2"})
_, err = tx.Exec(ctx, `
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
)
SELECT id, 'task.billing.settle', 'settle', $2::numeric, $3, $4::jsonb, $5::jsonb, 'pending', now()
FROM gateway_tasks
WHERE id = $1::uuid
AND run_mode = 'production'
AND gateway_user_id IS NOT NULL
ON CONFLICT (task_id, event_type) DO NOTHING`,
input.TaskID, finalChargeAmount, currency, string(pricingSnapshotJSON), string(payloadJSON))
return err
})
if err != nil {
return GatewayTask{}, err
}
return s.GetTask(ctx, input.TaskID)
}
func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManualReviewInput) (GatewayTask, error) {
status := strings.TrimSpace(input.TaskStatus)
if status != "succeeded" && status != "failed" {
return GatewayTask{}, fmt.Errorf("manual review task status must be succeeded or failed")
}
resultJSON, _ := json.Marshal(emptyObjectIfNil(input.Result))
pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot))
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if strings.TrimSpace(input.AttemptID) != "" {
attemptStatus := "failed"
if status == "succeeded" {
attemptStatus = "succeeded"
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_task_attempts
SET status = $2,
request_id = NULLIF($3, ''),
error_code = CASE WHEN $2 = 'failed' THEN NULLIF($4, '') ELSE NULL END,
error_message = CASE WHEN $2 = 'failed' THEN NULLIF($5, '') ELSE NULL END,
upstream_submission_status = CASE WHEN $2 = 'succeeded' THEN 'response_received' ELSE upstream_submission_status END,
upstream_submission_updated_at = now(),
response_started_at = $6::timestamptz,
response_finished_at = $7::timestamptz,
response_duration_ms = $8,
finished_at = now()
WHERE id = $1::uuid`, input.AttemptID, attemptStatus, input.RequestID, input.Code, input.Message,
nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS); err != nil {
return err
}
}
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET status = $2,
result = $3::jsonb,
request_id = NULLIF($4, ''),
error = CASE WHEN $2 = 'failed' THEN NULLIF($6, '') ELSE NULL END,
error_code = NULLIF($5, ''),
error_message = NULLIF($6, ''),
billing_status = 'manual_review',
pricing_snapshot = CASE WHEN $7::jsonb = '{}'::jsonb THEN pricing_snapshot ELSE $7::jsonb END,
request_fingerprint = COALESCE(NULLIF($8, ''), request_fingerprint),
billing_updated_at = now(),
response_started_at = $9::timestamptz,
response_finished_at = $10::timestamptz,
response_duration_ms = $11,
locked_by = NULL,
locked_at = NULL,
heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(),
updated_at = now()
WHERE id = $1::uuid
AND status = 'running'
AND execution_token = $12::uuid`, input.TaskID, status, string(resultJSON), input.RequestID, input.Code, input.Message,
string(pricingSnapshotJSON), input.RequestFingerprint, nullableTime(input.ResponseStartedAt),
nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS, input.ExecutionToken)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return ErrTaskExecutionLeaseLost
}
payloadJSON, _ := json.Marshal(map[string]any{
"taskId": input.TaskID, "classification": input.Code,
})
_, err = tx.Exec(ctx, `
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
status, next_attempt_at, manual_review_reason
)
SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency,
pricing_snapshot, $2::jsonb, 'manual_review', now(), $3
FROM gateway_tasks
WHERE id = $1::uuid
AND run_mode = 'production'
AND gateway_user_id IS NOT NULL
ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON), input.Code)
return err
})
if err != nil {
return GatewayTask{}, err return GatewayTask{}, err
} }
return s.GetTask(ctx, input.TaskID) return s.GetTask(ctx, input.TaskID)
@@ -743,6 +1088,9 @@ ON CONFLICT (gateway_user_id, currency) DO NOTHING`,
task.GatewayTenantID, task.GatewayUserID, task.TenantID, task.TenantKey, task.UserID, currency); err != nil { task.GatewayTenantID, task.GatewayUserID, task.TenantID, task.TenantKey, task.UserID, currency); err != nil {
return err return err
} }
if err := ensureWalletAccountAuditGuard(ctx, tx, task.GatewayUserID, currency); err != nil {
return err
}
var exists bool var exists bool
var accountID string var accountID string
var balanceBefore float64 var balanceBefore float64
@@ -859,7 +1207,8 @@ func taskBillingString(value any) string {
func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) { func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) {
metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics))
resultJSON, _ := json.Marshal(emptyObjectIfNil(input.Result)) resultJSON, _ := json.Marshal(emptyObjectIfNil(input.Result))
if _, err := s.pool.Exec(ctx, ` err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
tag, err := tx.Exec(ctx, `
UPDATE gateway_tasks UPDATE gateway_tasks
SET status = 'failed', SET status = 'failed',
error = NULLIF($2::text, ''), error = NULLIF($2::text, ''),
@@ -871,22 +1220,55 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn
response_finished_at = $7::timestamptz, response_finished_at = $7::timestamptz,
response_duration_ms = $8, response_duration_ms = $8,
result = $9::jsonb, result = $9::jsonb,
billing_status = CASE
WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required'
WHEN reservation_amount > 0 THEN 'pending'
ELSE 'released'
END,
billing_updated_at = now(),
locked_by = NULL, locked_by = NULL,
locked_at = NULL, locked_at = NULL,
heartbeat_at = NULL, heartbeat_at = NULL,
execution_token = NULL,
execution_lease_expires_at = NULL,
finished_at = now(), finished_at = now(),
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, WHERE id = $1::uuid
input.TaskID, AND status = 'running'
input.Message, AND execution_token = $10::uuid`,
input.Code, input.TaskID,
input.RequestID, input.Message,
string(metricsJSON), input.Code,
nullableTime(input.ResponseStartedAt), input.RequestID,
nullableTime(input.ResponseFinishedAt), string(metricsJSON),
input.ResponseDurationMS, nullableTime(input.ResponseStartedAt),
string(resultJSON), nullableTime(input.ResponseFinishedAt),
); err != nil { input.ResponseDurationMS,
string(resultJSON),
input.ExecutionToken,
)
if err != nil {
return err
}
if tag.RowsAffected() != 1 {
return ErrTaskExecutionLeaseLost
}
payloadJSON, _ := json.Marshal(map[string]any{"taskId": input.TaskID, "reason": input.Code})
_, err = tx.Exec(ctx, `
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at
)
SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency,
pricing_snapshot, $2::jsonb, 'pending', now()
FROM gateway_tasks
WHERE id = $1::uuid
AND run_mode = 'production'
AND gateway_user_id IS NOT NULL
AND reservation_amount > 0
ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON))
return err
})
if err != nil {
return GatewayTask{}, err return GatewayTask{}, err
} }
return s.GetTask(ctx, input.TaskID) return s.GetTask(ctx, input.TaskID)
apps/api/internal/store/wallet.go
+334 -60
View File
@@ -3,6 +3,7 @@ package store
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
@@ -82,6 +83,7 @@ type WalletBalanceAdjustmentInput struct {
GatewayUserID string `json:"gatewayUserId"` GatewayUserID string `json:"gatewayUserId"`
Currency string `json:"currency"` Currency string `json:"currency"`
Balance float64 `json:"balance"` Balance float64 `json:"balance"`
BalanceText string `json:"-"`
Reason string `json:"reason"` Reason string `json:"reason"`
IdempotencyKey string `json:"idempotencyKey"` IdempotencyKey string `json:"idempotencyKey"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
@@ -91,6 +93,7 @@ type WalletRechargeInput struct {
GatewayUserID string `json:"gatewayUserId"` GatewayUserID string `json:"gatewayUserId"`
Currency string `json:"currency"` Currency string `json:"currency"`
Amount float64 `json:"amount"` Amount float64 `json:"amount"`
AmountText string `json:"-"`
Reason string `json:"reason"` Reason string `json:"reason"`
IdempotencyKey string `json:"idempotencyKey"` IdempotencyKey string `json:"idempotencyKey"`
Metadata map[string]any `json:"metadata"` Metadata map[string]any `json:"metadata"`
@@ -135,7 +138,7 @@ func (s *Store) WalletAvailability(ctx context.Context, user *auth.User, currenc
return result, nil return result, nil
} }
func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *auth.User, billings []any) ([]WalletBillingReservation, error) { func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *auth.User, billings []any, pricingSnapshots ...map[string]any) ([]WalletBillingReservation, error) {
gatewayUserID := taskGatewayUserID(task, user) gatewayUserID := taskGatewayUserID(task, user)
if gatewayUserID == "" { if gatewayUserID == "" {
return nil, nil return nil, nil
@@ -145,12 +148,21 @@ func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *
return nil, fmt.Errorf("task id is required for wallet reservation") return nil, fmt.Errorf("task id is required for wallet reservation")
} }
pricingSnapshot := map[string]any{}
if len(pricingSnapshots) > 0 {
pricingSnapshot = emptyObjectIfNil(pricingSnapshots[0])
}
if exactAmount := walletString(pricingSnapshot["reservationAmount"]); exactAmount != "" {
return s.reserveTaskBillingExact(ctx, task, gatewayUserID, exactAmount, pricingSnapshot)
}
amounts := walletBillingAmounts(billings) amounts := walletBillingAmounts(billings)
if len(amounts) == 0 { if len(amounts) == 0 {
return nil, nil return nil, nil
} }
reservations := make([]WalletBillingReservation, 0, len(amounts)) reservations := make([]WalletBillingReservation, 0, len(amounts))
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
for currency, rawAmount := range amounts { for currency, rawAmount := range amounts {
amount := roundMoney(rawAmount) amount := roundMoney(rawAmount)
@@ -257,7 +269,30 @@ VALUES (
} }
reservations = append(reservations, reservation) reservations = append(reservations, reservation)
} }
return nil _, err := tx.Exec(ctx, `
UPDATE gateway_tasks task
SET reservation_amount = COALESCE((
SELECT SUM(reserve.amount)
FROM gateway_wallet_transactions reserve
WHERE reserve.reference_type = 'gateway_task'
AND reserve.reference_id = task.id::text
AND reserve.transaction_type = 'reserve'
AND NOT EXISTS (
SELECT 1
FROM gateway_wallet_transactions release
WHERE release.account_id = reserve.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = reserve.idempotency_key || ':release'
)
), 0),
billing_status = 'pending',
billing_currency = 'resource',
pricing_snapshot = $2::jsonb,
request_fingerprint = NULLIF($3, ''),
billing_updated_at = now(),
updated_at = now()
WHERE task.id = $1::uuid`, taskID, string(pricingSnapshotJSON), requestFingerprint)
return err
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -265,6 +300,106 @@ VALUES (
return reservations, err return reservations, err
} }
func (s *Store) reserveTaskBillingExact(ctx context.Context, task GatewayTask, gatewayUserID string, amount string, pricingSnapshot map[string]any) ([]WalletBillingReservation, error) {
taskID := strings.TrimSpace(task.ID)
currency := normalizeWalletCurrency(walletString(pricingSnapshot["currency"]))
if currency != "resource" {
return nil, fmt.Errorf("unsupported billing currency %q", currency)
}
pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot)
requestFingerprint := walletString(pricingSnapshot["requestFingerprint"])
var reservations []WalletBillingReservation
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var positive bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric(38, 9) > 0`, amount).Scan(&positive); err != nil {
return fmt.Errorf("invalid exact reservation amount: %w", err)
}
if !positive {
return nil
}
account, err := s.ensureWalletAccount(ctx, tx, gatewayUserID, currency)
if err != nil {
return err
}
locked, err := lockWalletAccount(ctx, tx, account.ID)
if err != nil {
return err
}
activeKey, activeAmount, err := activeWalletReservationExact(ctx, tx, locked.ID, taskID)
if err != nil {
return err
}
if activeAmount != "" {
presentationAmount, _ := strconv.ParseFloat(activeAmount, 64)
reservations = []WalletBillingReservation{{
TaskID: taskID, AccountID: locked.ID, GatewayUserID: gatewayUserID,
GatewayTenantID: firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID),
Currency: locked.Currency, Amount: presentationAmount, IdempotencyKey: activeKey,
}}
return nil
}
sequence, err := nextWalletReservationSequence(ctx, tx, locked.ID, taskID)
if err != nil {
return err
}
key := billingReservationIdempotencyKey(taskID, locked.Currency, sequence)
var balanceBefore string
var frozenBefore string
if err := tx.QueryRow(ctx, `
SELECT balance::text, frozen_balance::text
FROM gateway_wallet_accounts
WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore, &frozenBefore); err != nil {
return err
}
var frozenAfter string
err = tx.QueryRow(ctx, `
UPDATE gateway_wallet_accounts
SET frozen_balance = frozen_balance + $2::numeric(38, 9), updated_at = now()
WHERE id = $1::uuid
AND balance - frozen_balance >= $2::numeric(38, 9)
RETURNING frozen_balance::text`, locked.ID, amount).Scan(&frozenAfter)
if errors.Is(err, pgx.ErrNoRows) {
return ErrInsufficientWalletBalance
}
if err != nil {
return err
}
metadata, _ := json.Marshal(map[string]any{
"taskId": taskID, "reserved": amount, "balance": balanceBefore,
"frozenBefore": frozenBefore, "frozenAfter": frozenAfter,
"pricingVersion": walletString(pricingSnapshot["pricingVersion"]),
})
if _, err := tx.Exec(ctx, `
INSERT INTO gateway_wallet_transactions (
account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type,
amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata
)
VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'debit', 'reserve',
$4::numeric(38, 9), $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb
)`, locked.ID, firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID), gatewayUserID,
amount, balanceBefore, key, taskID, string(metadata)); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks
SET reservation_amount = $2::numeric(38, 9), billing_status = 'pending',
billing_currency = $3, pricing_snapshot = $4::jsonb,
request_fingerprint = NULLIF($5, ''), billing_updated_at = now(), updated_at = now()
WHERE id = $1::uuid`, taskID, amount, currency, string(pricingSnapshotJSON), requestFingerprint); err != nil {
return err
}
presentationAmount, _ := strconv.ParseFloat(amount, 64)
reservations = []WalletBillingReservation{{
TaskID: taskID, AccountID: locked.ID, GatewayUserID: gatewayUserID,
GatewayTenantID: firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID),
Currency: locked.Currency, Amount: presentationAmount, IdempotencyKey: key,
}}
return nil
})
return reservations, err
}
func (s *Store) ReleaseTaskBillingReservations(ctx context.Context, reservations []WalletBillingReservation, reason string) error { func (s *Store) ReleaseTaskBillingReservations(ctx context.Context, reservations []WalletBillingReservation, reason string) error {
if len(reservations) == 0 { if len(reservations) == 0 {
return nil return nil
@@ -274,10 +409,14 @@ func (s *Store) ReleaseTaskBillingReservations(ctx context.Context, reservations
reason = "task_not_settled" reason = "task_not_settled"
} }
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
taskIDs := map[string]struct{}{}
for _, reservation := range reservations { for _, reservation := range reservations {
if reservation.Amount <= 0 || strings.TrimSpace(reservation.AccountID) == "" { if strings.TrimSpace(reservation.AccountID) == "" {
continue continue
} }
if taskID := strings.TrimSpace(reservation.TaskID); taskID != "" {
taskIDs[taskID] = struct{}{}
}
reserveKey := strings.TrimSpace(reservation.IdempotencyKey) reserveKey := strings.TrimSpace(reservation.IdempotencyKey)
if reserveKey == "" { if reserveKey == "" {
reserveKey = billingReservationIdempotencyKey(reservation.TaskID, reservation.Currency, 1) reserveKey = billingReservationIdempotencyKey(reservation.TaskID, reservation.Currency, 1)
@@ -303,39 +442,44 @@ SELECT EXISTS (
if alreadyReleased { if alreadyReleased {
continue continue
} }
var storedReservedAmount float64 var storedReservedAmount string
if err := tx.QueryRow(ctx, ` if err := tx.QueryRow(ctx, `
SELECT COALESCE(( SELECT COALESCE((
SELECT amount::float8 SELECT amount::text
FROM gateway_wallet_transactions FROM gateway_wallet_transactions
WHERE account_id = $1::uuid WHERE account_id = $1::uuid
AND idempotency_key = $2 AND idempotency_key = $2
AND transaction_type = 'reserve' AND transaction_type = 'reserve'
LIMIT 1 LIMIT 1
), 0)::float8`, reservation.AccountID, reserveKey).Scan(&storedReservedAmount); err != nil { ), '0')`, reservation.AccountID, reserveKey).Scan(&storedReservedAmount); err != nil {
return err return err
} }
if storedReservedAmount <= 0 { var positive bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, storedReservedAmount).Scan(&positive); err != nil {
return err
}
if !positive {
continue continue
} }
amount := roundMoney(storedReservedAmount) var balanceBefore string
frozenAfter := roundMoney(locked.FrozenBalance - amount) var frozenBefore string
if frozenAfter < 0 { var frozenAfter string
frozenAfter = 0 if err := tx.QueryRow(ctx, `
}
if _, err := tx.Exec(ctx, `
UPDATE gateway_wallet_accounts UPDATE gateway_wallet_accounts
SET frozen_balance = $2, SET frozen_balance = frozen_balance - $2::numeric,
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, locked.ID, frozenAfter); err != nil { WHERE id = $1::uuid
AND frozen_balance >= $2::numeric
RETURNING balance::text, (frozen_balance + $2::numeric)::text, frozen_balance::text`,
locked.ID, storedReservedAmount).Scan(&balanceBefore, &frozenBefore, &frozenAfter); err != nil {
return err return err
} }
metadata, _ := json.Marshal(map[string]any{ metadata, _ := json.Marshal(map[string]any{
"taskId": reservation.TaskID, "taskId": reservation.TaskID,
"reason": reason, "reason": reason,
"reserved": amount, "reserved": storedReservedAmount,
"frozenBefore": roundMoney(locked.FrozenBalance), "frozenBefore": frozenBefore,
"frozenAfter": frozenAfter, "frozenAfter": frozenAfter,
}) })
if _, err := tx.Exec(ctx, ` if _, err := tx.Exec(ctx, `
@@ -345,15 +489,14 @@ INSERT INTO gateway_wallet_transactions (
) )
VALUES ( VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release', $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release',
$4, $5, $6, $7, 'gateway_task', $8, $9::jsonb $4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb
) )
ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`, ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`,
locked.ID, locked.ID,
locked.GatewayTenantID, locked.GatewayTenantID,
locked.GatewayUserID, locked.GatewayUserID,
amount, storedReservedAmount,
roundMoney(locked.Balance), balanceBefore,
roundMoney(locked.Balance),
releaseKey, releaseKey,
reservation.TaskID, reservation.TaskID,
string(metadata), string(metadata),
@@ -361,6 +504,33 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N
return err return err
} }
} }
for taskID := range taskIDs {
if _, err := tx.Exec(ctx, `
UPDATE gateway_tasks task
SET reservation_amount = COALESCE((
SELECT SUM(reserve.amount)
FROM gateway_wallet_transactions reserve
WHERE reserve.reference_type = 'gateway_task'
AND reserve.reference_id = task.id::text
AND reserve.transaction_type = 'reserve'
AND NOT EXISTS (
SELECT 1 FROM gateway_wallet_transactions release
WHERE release.account_id = reserve.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = reserve.idempotency_key || ':release'
)
), 0),
billing_status = CASE
WHEN status IN ('failed', 'cancelled') THEN 'released'
WHEN status = 'succeeded' THEN billing_status
ELSE 'not_started'
END,
billing_updated_at = now(),
updated_at = now()
WHERE id = $1::uuid`, taskID); err != nil {
return err
}
}
return nil return nil
}) })
} }
@@ -588,8 +758,20 @@ func (s *Store) SetUserWalletBalanceTx(ctx context.Context, tx Tx, input WalletB
if input.GatewayUserID == "" { if input.GatewayUserID == "" {
return WalletAdjustmentResult{}, ErrLocalUserRequired return WalletAdjustmentResult{}, ErrLocalUserRequired
} }
if input.Balance < 0 { targetInput := strings.TrimSpace(input.BalanceText)
return WalletAdjustmentResult{}, fmt.Errorf("wallet balance cannot be negative") if targetInput == "" {
targetInput = strconv.FormatFloat(input.Balance, 'f', 9, 64)
}
targetBalance, err := canonicalWalletAmount(ctx, tx, targetInput)
if err != nil {
return WalletAdjustmentResult{}, err
}
var nonnegative bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric >= 0`, targetBalance).Scan(&nonnegative); err != nil {
return WalletAdjustmentResult{}, err
}
if !nonnegative {
return WalletAdjustmentResult{}, ErrInvalidWalletAmount
} }
account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency)
if err != nil { if err != nil {
@@ -608,38 +790,68 @@ FOR UPDATE`, account.ID))
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
before := locked before := locked
nextBalance := roundMoney(input.Balance) var balanceBefore string
delta := roundMoney(nextBalance - locked.Balance) var frozenBalance string
if delta == 0 { if err := tx.QueryRow(ctx, `
SELECT balance::text, frozen_balance::text
FROM gateway_wallet_accounts
WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore, &frozenBalance); err != nil {
return WalletAdjustmentResult{}, err
}
var unchanged bool
var belowFrozen bool
if err := tx.QueryRow(ctx, `
SELECT $1::numeric = $2::numeric, $1::numeric < $3::numeric`,
targetBalance, balanceBefore, frozenBalance).Scan(&unchanged, &belowFrozen); err != nil {
return WalletAdjustmentResult{}, err
}
if unchanged {
return WalletAdjustmentResult{}, ErrWalletBalanceUnchanged return WalletAdjustmentResult{}, ErrWalletBalanceUnchanged
} }
if belowFrozen {
return WalletAdjustmentResult{}, ErrBalanceBelowFrozen
}
direction := "credit" direction := "credit"
amount := delta if strings.HasPrefix(balanceBefore, "-") {
if delta < 0 { return WalletAdjustmentResult{}, ErrInvalidWalletAmount
}
var targetGreater bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric > $2::numeric`, targetBalance, balanceBefore).Scan(&targetGreater); err != nil {
return WalletAdjustmentResult{}, err
}
if !targetGreater {
direction = "debit" direction = "debit"
amount = -delta }
var amount string
if err := tx.QueryRow(ctx, `SELECT abs($1::numeric - $2::numeric)::text`, targetBalance, balanceBefore).Scan(&amount); err != nil {
return WalletAdjustmentResult{}, err
} }
reason := strings.TrimSpace(input.Reason) reason := strings.TrimSpace(input.Reason)
if reason == "" { if reason == "" {
reason = "后台余额调整" reason = "后台余额调整"
} }
if _, err := tx.Exec(ctx, ` tag, err := tx.Exec(ctx, `
UPDATE gateway_wallet_accounts UPDATE gateway_wallet_accounts
SET balance = $2, SET balance = $2::numeric,
total_recharged = total_recharged + CASE WHEN $3 = 'credit' THEN $4 ELSE 0 END, total_recharged = total_recharged + CASE WHEN $3 = 'credit' THEN $4::numeric ELSE 0 END,
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, WHERE id = $1::uuid
AND $2::numeric >= frozen_balance`,
locked.ID, locked.ID,
nextBalance, targetBalance,
direction, direction,
amount, amount,
); err != nil { )
if err != nil {
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
if tag.RowsAffected() != 1 {
return WalletAdjustmentResult{}, ErrBalanceBelowFrozen
}
metadata := mergeObjects(input.Metadata, map[string]any{ metadata := mergeObjects(input.Metadata, map[string]any{
"reason": reason, "reason": reason,
"previousBalance": roundMoney(before.Balance), "previousBalance": balanceBefore,
"targetBalance": nextBalance, "targetBalance": targetBalance,
}) })
metadataJSON, _ := json.Marshal(emptyObjectIfNil(metadata)) metadataJSON, _ := json.Marshal(emptyObjectIfNil(metadata))
transaction, err := scanWalletTransaction(tx.QueryRow(ctx, ` transaction, err := scanWalletTransaction(tx.QueryRow(ctx, `
@@ -649,7 +861,7 @@ INSERT INTO gateway_wallet_transactions (
) )
VALUES ( VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, $4, 'admin_adjust', $1::uuid, NULLIF($2, '')::uuid, $3::uuid, $4, 'admin_adjust',
$5, $6, $7, NULLIF($8, ''), 'gateway_user', $9, $10::jsonb $5::numeric, $6::numeric, $7::numeric, NULLIF($8, ''), 'gateway_user', $9, $10::jsonb
) )
RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''), RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''),
direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8, direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8,
@@ -660,8 +872,8 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA
locked.GatewayUserID, locked.GatewayUserID,
direction, direction,
amount, amount,
roundMoney(before.Balance), balanceBefore,
nextBalance, targetBalance,
strings.TrimSpace(input.IdempotencyKey), strings.TrimSpace(input.IdempotencyKey),
locked.GatewayUserID, locked.GatewayUserID,
string(metadataJSON), string(metadataJSON),
@@ -669,11 +881,10 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA
if err != nil { if err != nil {
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
locked.Balance = nextBalance locked, err = s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency)
if direction == "credit" { if err != nil {
locked.TotalRecharged = roundMoney(locked.TotalRecharged + amount) return WalletAdjustmentResult{}, err
} }
locked.UpdatedAt = time.Now()
return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil
} }
@@ -695,9 +906,20 @@ func (s *Store) RechargeUserWalletBalanceTx(ctx context.Context, tx Tx, input Wa
if input.GatewayUserID == "" { if input.GatewayUserID == "" {
return WalletAdjustmentResult{}, ErrLocalUserRequired return WalletAdjustmentResult{}, ErrLocalUserRequired
} }
amount := roundMoney(input.Amount) amountInput := strings.TrimSpace(input.AmountText)
if amount <= 0 { if amountInput == "" {
return WalletAdjustmentResult{}, fmt.Errorf("wallet recharge amount must be positive") amountInput = strconv.FormatFloat(input.Amount, 'f', 9, 64)
}
amount, err := canonicalWalletAmount(ctx, tx, amountInput)
if err != nil {
return WalletAdjustmentResult{}, err
}
var positive bool
if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, amount).Scan(&positive); err != nil {
return WalletAdjustmentResult{}, err
}
if !positive {
return WalletAdjustmentResult{}, ErrInvalidWalletAmount
} }
account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency)
if err != nil { if err != nil {
@@ -715,26 +937,30 @@ FOR UPDATE`, account.ID))
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
before := locked before := locked
nextBalance := roundMoney(locked.Balance + amount) var balanceBefore string
if err := tx.QueryRow(ctx, `SELECT balance::text FROM gateway_wallet_accounts WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore); err != nil {
return WalletAdjustmentResult{}, err
}
reason := strings.TrimSpace(input.Reason) reason := strings.TrimSpace(input.Reason)
if reason == "" { if reason == "" {
reason = "后台余额充值" reason = "后台余额充值"
} }
if _, err := tx.Exec(ctx, ` var nextBalance string
if err := tx.QueryRow(ctx, `
UPDATE gateway_wallet_accounts UPDATE gateway_wallet_accounts
SET balance = $2, SET balance = balance + $2::numeric,
total_recharged = total_recharged + $3, total_recharged = total_recharged + $2::numeric,
updated_at = now() updated_at = now()
WHERE id = $1::uuid`, WHERE id = $1::uuid
RETURNING balance::text`,
locked.ID, locked.ID,
nextBalance,
amount, amount,
); err != nil { ).Scan(&nextBalance); err != nil {
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
metadata := mergeObjects(input.Metadata, map[string]any{ metadata := mergeObjects(input.Metadata, map[string]any{
"reason": reason, "reason": reason,
"previousBalance": roundMoney(before.Balance), "previousBalance": balanceBefore,
"rechargeAmount": amount, "rechargeAmount": amount,
"targetBalance": nextBalance, "targetBalance": nextBalance,
}) })
@@ -746,7 +972,7 @@ INSERT INTO gateway_wallet_transactions (
) )
VALUES ( VALUES (
$1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'recharge', $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'recharge',
$4, $5, $6, NULLIF($7, ''), 'gateway_user', $8, $9::jsonb $4::numeric, $5::numeric, $6::numeric, NULLIF($7, ''), 'gateway_user', $8, $9::jsonb
) )
RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''), RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''),
direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8, direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8,
@@ -756,7 +982,7 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA
locked.GatewayTenantID, locked.GatewayTenantID,
locked.GatewayUserID, locked.GatewayUserID,
amount, amount,
roundMoney(before.Balance), balanceBefore,
nextBalance, nextBalance,
strings.TrimSpace(input.IdempotencyKey), strings.TrimSpace(input.IdempotencyKey),
locked.GatewayUserID, locked.GatewayUserID,
@@ -765,12 +991,21 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA
if err != nil { if err != nil {
return WalletAdjustmentResult{}, err return WalletAdjustmentResult{}, err
} }
locked.Balance = nextBalance locked, err = s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency)
locked.TotalRecharged = roundMoney(locked.TotalRecharged + amount) if err != nil {
locked.UpdatedAt = time.Now() return WalletAdjustmentResult{}, err
}
return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil
} }
func canonicalWalletAmount(ctx context.Context, q Tx, value string) (string, error) {
var canonical string
if err := q.QueryRow(ctx, `SELECT $1::numeric(38, 9)::text`, strings.TrimSpace(value)).Scan(&canonical); err != nil {
return "", fmt.Errorf("%w: %v", ErrInvalidWalletAmount, err)
}
return canonical, nil
}
func (s *Store) ensureWalletAccount(ctx context.Context, q Tx, gatewayUserID string, currency string) (GatewayWalletAccount, error) { func (s *Store) ensureWalletAccount(ctx context.Context, q Tx, gatewayUserID string, currency string) (GatewayWalletAccount, error) {
currency = normalizeWalletCurrency(currency) currency = normalizeWalletCurrency(currency)
if _, err := q.Exec(ctx, ` if _, err := q.Exec(ctx, `
@@ -794,6 +1029,9 @@ WHERE gateway_wallet_accounts.gateway_tenant_id IS NULL
OR COALESCE(gateway_wallet_accounts.user_id, '') = ''`, gatewayUserID, currency); err != nil { OR COALESCE(gateway_wallet_accounts.user_id, '') = ''`, gatewayUserID, currency); err != nil {
return GatewayWalletAccount{}, err return GatewayWalletAccount{}, err
} }
if err := ensureWalletAccountAuditGuard(ctx, q, gatewayUserID, currency); err != nil {
return GatewayWalletAccount{}, err
}
account, err := scanWalletAccount(q.QueryRow(ctx, ` account, err := scanWalletAccount(q.QueryRow(ctx, `
SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text, SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text,
COALESCE(tenant_id, ''), COALESCE(tenant_key, ''), COALESCE(user_id, ''), COALESCE(tenant_id, ''), COALESCE(tenant_key, ''), COALESCE(user_id, ''),
@@ -811,6 +1049,17 @@ WHERE gateway_user_id = $1::uuid
return account, nil return account, nil
} }
func ensureWalletAccountAuditGuard(ctx context.Context, q Tx, gatewayUserID string, currency string) error {
_, err := q.Exec(ctx, `
INSERT INTO gateway_wallet_account_audit_guards (account_id)
SELECT id
FROM gateway_wallet_accounts
WHERE gateway_user_id = $1::uuid
AND currency = $2
ON CONFLICT (account_id) DO NOTHING`, gatewayUserID, normalizeWalletCurrency(currency))
return err
}
func lockWalletAccount(ctx context.Context, tx pgx.Tx, accountID string) (GatewayWalletAccount, error) { func lockWalletAccount(ctx context.Context, tx pgx.Tx, accountID string) (GatewayWalletAccount, error) {
return scanWalletAccount(tx.QueryRow(ctx, ` return scanWalletAccount(tx.QueryRow(ctx, `
SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text, SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text,
@@ -851,6 +1100,31 @@ LIMIT 1`, accountID, taskID).Scan(&key, &amount)
return key, roundMoney(amount), nil return key, roundMoney(amount), nil
} }
func activeWalletReservationExact(ctx context.Context, tx pgx.Tx, accountID string, taskID string) (string, string, error) {
var key string
var amount string
err := tx.QueryRow(ctx, `
SELECT COALESCE(t.idempotency_key, ''), t.amount::text
FROM gateway_wallet_transactions t
WHERE t.account_id = $1::uuid
AND t.reference_type = 'gateway_task'
AND t.reference_id = $2
AND t.transaction_type = 'reserve'
AND COALESCE(t.idempotency_key, '') <> ''
AND NOT EXISTS (
SELECT 1 FROM gateway_wallet_transactions release
WHERE release.account_id = t.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = t.idempotency_key || ':release'
)
ORDER BY t.created_at DESC
LIMIT 1`, accountID, taskID).Scan(&key, &amount)
if errors.Is(err, pgx.ErrNoRows) {
return "", "", nil
}
return key, amount, err
}
func nextWalletReservationSequence(ctx context.Context, tx pgx.Tx, accountID string, taskID string) (int, error) { func nextWalletReservationSequence(ctx context.Context, tx pgx.Tx, accountID string, taskID string) (int, error) {
var count int var count int
if err := tx.QueryRow(ctx, ` if err := tx.QueryRow(ctx, `
apps/api/internal/store/wallet_reservation_test.go
+4
View File
@@ -132,6 +132,10 @@ RETURNING id::text`, "wallet-reservation-user-"+suffix, "wallet_reservation_"+su
} }
t.Cleanup(func() { t.Cleanup(func() {
cleanupCtx := context.Background() cleanupCtx := context.Background()
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id = $1::uuid`, userID)
_, _ = db.pool.Exec(cleanupCtx, `
DELETE FROM gateway_wallet_account_audit_guards
WHERE account_id IN (SELECT id FROM gateway_wallet_accounts WHERE gateway_user_id = $1::uuid)`, userID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_users WHERE id = $1::uuid`, userID) _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_users WHERE id = $1::uuid`, userID)
_, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tenants WHERE id = $1::uuid`, tenantID) _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tenants WHERE id = $1::uuid`, tenantID)
}) })
apps/api/migrations/0069_billing_correctness_v2.sql
+234
View File
@@ -0,0 +1,234 @@
ALTER TABLE gateway_tasks
ADD COLUMN IF NOT EXISTS billing_version text DEFAULT 'effective-pricing-v2',
ADD COLUMN IF NOT EXISTS billing_status text DEFAULT 'not_started',
ADD COLUMN IF NOT EXISTS billing_currency text DEFAULT 'resource',
ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb,
ADD COLUMN IF NOT EXISTS request_fingerprint text,
ADD COLUMN IF NOT EXISTS idempotency_key_hash text,
ADD COLUMN IF NOT EXISTS idempotency_request_hash text,
ADD COLUMN IF NOT EXISTS reservation_amount numeric(38, 9) DEFAULT 0,
ADD COLUMN IF NOT EXISTS execution_token uuid,
ADD COLUMN IF NOT EXISTS execution_lease_expires_at timestamptz,
ADD COLUMN IF NOT EXISTS billing_updated_at timestamptz,
ADD COLUMN IF NOT EXISTS billing_settled_at timestamptz;
UPDATE gateway_tasks
SET billing_version = COALESCE(billing_version, 'effective-pricing-v2'),
billing_status = COALESCE(billing_status, 'not_started'),
billing_currency = COALESCE(billing_currency, 'resource'),
pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb),
reservation_amount = COALESCE(reservation_amount, 0);
ALTER TABLE gateway_tasks
ADD CONSTRAINT gateway_tasks_billing_status_v2_check CHECK (
billing_status IN (
'not_started', 'pending', 'processing', 'settled', 'released',
'retryable_failed', 'manual_review', 'not_required'
)
) NOT VALID,
ADD CONSTRAINT gateway_tasks_reservation_amount_v2_check CHECK (reservation_amount >= 0) NOT VALID;
CREATE UNIQUE INDEX IF NOT EXISTS uniq_gateway_tasks_idempotency_hash_v2
ON gateway_tasks(user_id, idempotency_key_hash)
WHERE idempotency_key_hash IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_gateway_tasks_execution_lease_v2
ON gateway_tasks(status, execution_lease_expires_at)
WHERE status = 'running';
CREATE INDEX IF NOT EXISTS idx_gateway_tasks_billing_status_v2
ON gateway_tasks(billing_status, billing_updated_at, created_at);
ALTER TABLE gateway_task_attempts
ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb,
ADD COLUMN IF NOT EXISTS request_fingerprint text,
ADD COLUMN IF NOT EXISTS upstream_submission_status text DEFAULT 'not_submitted',
ADD COLUMN IF NOT EXISTS upstream_submission_updated_at timestamptz;
UPDATE gateway_task_attempts
SET pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb),
upstream_submission_status = COALESCE(upstream_submission_status, 'not_submitted');
ALTER TABLE gateway_task_attempts
ADD CONSTRAINT gateway_task_attempts_submission_v2_check CHECK (
upstream_submission_status IN ('not_submitted', 'submitting', 'response_received')
) NOT VALID;
ALTER TABLE settlement_outbox
ADD COLUMN IF NOT EXISTS action text DEFAULT 'settle',
ADD COLUMN IF NOT EXISTS amount numeric(38, 9) DEFAULT 0,
ADD COLUMN IF NOT EXISTS currency text DEFAULT 'resource',
ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb,
ADD COLUMN IF NOT EXISTS locked_by text,
ADD COLUMN IF NOT EXISTS lock_token uuid,
ADD COLUMN IF NOT EXISTS locked_at timestamptz,
ADD COLUMN IF NOT EXISTS last_error_code text,
ADD COLUMN IF NOT EXISTS last_error_message text,
ADD COLUMN IF NOT EXISTS completed_at timestamptz,
ADD COLUMN IF NOT EXISTS manual_review_reason text,
ADD COLUMN IF NOT EXISTS retry_idempotency_key_hash text,
ADD COLUMN IF NOT EXISTS retry_requested_at timestamptz;
UPDATE settlement_outbox
SET action = COALESCE(action, 'settle'),
amount = COALESCE(amount, 0),
currency = COALESCE(currency, 'resource'),
pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb);
ALTER TABLE settlement_outbox
ADD CONSTRAINT settlement_outbox_action_v2_check CHECK (action IN ('settle', 'release')) NOT VALID,
ADD CONSTRAINT settlement_outbox_status_v2_check CHECK (
status IN ('pending', 'processing', 'retryable_failed', 'completed', 'manual_review')
) NOT VALID,
ADD CONSTRAINT settlement_outbox_amount_v2_check CHECK (amount >= 0) NOT VALID;
CREATE INDEX IF NOT EXISTS idx_settlement_outbox_claim_v2
ON settlement_outbox(status, next_attempt_at, created_at)
WHERE status IN ('pending', 'retryable_failed');
CREATE INDEX IF NOT EXISTS idx_settlement_outbox_stale_v2
ON settlement_outbox(status, locked_at)
WHERE status = 'processing';
ALTER TABLE model_pricing_rules
ADD COLUMN IF NOT EXISTS is_free boolean DEFAULT false;
UPDATE model_pricing_rules SET is_free = false WHERE is_free IS NULL;
UPDATE model_pricing_rules
SET calculator_type = CASE
WHEN resource_type IN ('text_input', 'text_cached_input', 'text_output', 'text_total') THEN 'token_usage'
WHEN resource_type = 'video' THEN 'duration_weight'
ELSE calculator_type
END,
unit = CASE
WHEN resource_type = 'video' AND unit = 'video' THEN '5s'
ELSE unit
END,
updated_at = now()
WHERE (resource_type IN ('text_input', 'text_cached_input', 'text_output', 'text_total') AND calculator_type = 'unit_weight')
OR (resource_type = 'video' AND (calculator_type = 'unit_weight' OR unit = 'video'));
ALTER TABLE model_pricing_rules
ADD CONSTRAINT model_pricing_rules_explicit_free_v2_check CHECK (
base_price > 0 OR (base_price = 0 AND is_free)
) NOT VALID;
ALTER TABLE gateway_wallet_accounts
ADD CONSTRAINT gateway_wallet_nonnegative_frozen_v2_check CHECK (frozen_balance >= 0) NOT VALID,
ADD CONSTRAINT gateway_wallet_balance_covers_frozen_v2_check CHECK (balance >= frozen_balance) NOT VALID;
CREATE TABLE IF NOT EXISTS gateway_wallet_account_audit_guards (
account_id uuid PRIMARY KEY REFERENCES gateway_wallet_accounts(id) ON DELETE RESTRICT,
created_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO gateway_wallet_account_audit_guards (account_id)
SELECT id FROM gateway_wallet_accounts
ON CONFLICT (account_id) DO NOTHING;
ALTER TABLE gateway_wallet_transactions
ADD CONSTRAINT gateway_wallet_transactions_audit_guard_v2
FOREIGN KEY (account_id) REFERENCES gateway_wallet_account_audit_guards(account_id) ON DELETE RESTRICT NOT VALID;
UPDATE gateway_tasks task
SET billing_status = 'not_required',
billing_updated_at = now()
WHERE task.run_mode <> 'production'
OR task.gateway_user_id IS NULL;
UPDATE gateway_tasks task
SET billing_status = 'settled',
billing_currency = COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'),
billing_updated_at = now(),
billing_settled_at = COALESCE(task.finished_at, task.updated_at)
WHERE task.status = 'succeeded'
AND EXISTS (
SELECT 1
FROM gateway_wallet_transactions transaction
WHERE transaction.reference_type = 'gateway_task'
AND transaction.reference_id = task.id::text
AND transaction.transaction_type = 'task_billing'
);
UPDATE gateway_tasks task
SET billing_status = 'manual_review',
billing_currency = COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'),
billing_updated_at = now()
WHERE task.status = 'succeeded'
AND task.run_mode = 'production'
AND task.gateway_user_id IS NOT NULL
AND NOT EXISTS (
SELECT 1
FROM gateway_wallet_transactions transaction
WHERE transaction.reference_type = 'gateway_task'
AND transaction.reference_id = task.id::text
AND transaction.transaction_type = 'task_billing'
);
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, pricing_snapshot, payload,
status, next_attempt_at, manual_review_reason
)
SELECT task.id,
'task.billing.review',
'settle',
COALESCE(task.final_charge_amount, 0),
COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'),
COALESCE(task.pricing_snapshot, '{}'::jsonb),
jsonb_build_object(
'taskId', task.id,
'classification', 'historical_success_without_charge'
),
'manual_review',
now(),
'historical_success_without_charge'
FROM gateway_tasks task
WHERE task.billing_status = 'manual_review'
AND task.status = 'succeeded'
AND task.run_mode = 'production'
AND task.gateway_user_id IS NOT NULL
ON CONFLICT (task_id, event_type) DO NOTHING;
INSERT INTO settlement_outbox (
task_id, event_type, action, amount, currency, payload, status, next_attempt_at
)
SELECT task.id,
'task.billing.release',
'release',
transaction.amount,
account.currency,
jsonb_build_object(
'taskId', task.id,
'reservationIdempotencyKey', transaction.idempotency_key,
'classification', 'historical_terminal_reservation'
),
'pending',
now()
FROM gateway_tasks task
JOIN gateway_wallet_transactions transaction
ON transaction.reference_type = 'gateway_task'
AND transaction.reference_id = task.id::text
AND transaction.transaction_type = 'reserve'
JOIN gateway_wallet_accounts account ON account.id = transaction.account_id
WHERE task.status IN ('failed', 'cancelled')
AND COALESCE(transaction.idempotency_key, '') <> ''
AND NOT EXISTS (
SELECT 1
FROM gateway_wallet_transactions release
WHERE release.account_id = transaction.account_id
AND release.transaction_type = 'release'
AND release.idempotency_key = transaction.idempotency_key || ':release'
)
ON CONFLICT (task_id, event_type) DO NOTHING;
UPDATE gateway_tasks task
SET billing_status = 'pending',
billing_updated_at = now()
WHERE EXISTS (
SELECT 1
FROM settlement_outbox outbox
WHERE outbox.task_id = task.id
AND outbox.action = 'release'
AND outbox.status IN ('pending', 'processing', 'retryable_failed')
);
apps/web/src/api.ts
+28 -1
View File
@@ -3,6 +3,8 @@ import type {
AuthUser, AuthUser,
BaseModelCatalogItem, BaseModelCatalogItem,
BaseModelUpsertRequest, BaseModelUpsertRequest,
BillingSettlementListResponse,
BillingSettlementRetryResponse,
CatalogProvider, CatalogProvider,
CatalogProviderUpsertRequest, CatalogProviderUpsertRequest,
ClientCustomizationSettings, ClientCustomizationSettings,
@@ -896,10 +898,12 @@ export async function uploadFileToStorage(
export async function estimatePricing( export async function estimatePricing(
token: string, token: string,
input: Record<string, unknown>, input: Record<string, unknown>,
signal?: AbortSignal,
): Promise<GatewayPricingEstimate> { ): Promise<GatewayPricingEstimate> {
return request<GatewayPricingEstimate>('/api/v1/pricing/estimate', { return request<GatewayPricingEstimate>('/api/v1/pricing/estimate', {
body: input, body: input,
method: 'POST', method: 'POST',
signal,
token, token,
}); });
} }
@@ -998,6 +1002,28 @@ export async function restoreModelRuntimeStatus(token: string, platformModelId:
}); });
} }
export async function listBillingSettlements(
token: string,
input: { status?: string; action?: string; page?: number; pageSize?: number; signal?: AbortSignal } = {},
): Promise<BillingSettlementListResponse> {
const search = new URLSearchParams({
page: String(input.page ?? 1),
pageSize: String(input.pageSize ?? 50),
});
if (input.status) search.set('status', input.status);
if (input.action) search.set('action', input.action);
return request<BillingSettlementListResponse>(`/api/admin/runtime/billing-settlements?${search.toString()}`, { token, signal: input.signal });
}
export async function retryBillingSettlement(token: string, settlementId: string): Promise<BillingSettlementRetryResponse> {
const idempotencyKey = crypto.randomUUID();
return request<BillingSettlementRetryResponse>(`/api/admin/runtime/billing-settlements/${settlementId}/retry`, {
method: 'POST',
token,
headers: { 'Idempotency-Key': idempotencyKey },
});
}
export async function getNetworkProxyConfig(token: string): Promise<GatewayNetworkProxyConfig> { export async function getNetworkProxyConfig(token: string): Promise<GatewayNetworkProxyConfig> {
return request<GatewayNetworkProxyConfig>('/api/admin/config/network-proxy', { token }); return request<GatewayNetworkProxyConfig>('/api/admin/config/network-proxy', { token });
} }
@@ -1180,7 +1206,7 @@ export async function deleteFileStorageChannel(token: string, channelId: string)
async function request<T>( async function request<T>(
path: string, path: string,
options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record<string, string> } = {}, options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record<string, string>; signal?: AbortSignal } = {},
): Promise<T> { ): Promise<T> {
const headers: Record<string, string> = { ...(options.headers ?? {}) }; const headers: Record<string, string> = { ...(options.headers ?? {}) };
if (options.auth !== false && options.token && options.token !== OIDC_BROWSER_SESSION_CREDENTIAL) { if (options.auth !== false && options.token && options.token !== OIDC_BROWSER_SESSION_CREDENTIAL) {
@@ -1194,6 +1220,7 @@ async function request<T>(
headers, headers,
body: options.body === undefined ? undefined : JSON.stringify(options.body), body: options.body === undefined ? undefined : JSON.stringify(options.body),
credentials: 'include', credentials: 'include',
signal: options.signal,
}); });
if (!response.ok) { if (!response.ok) {
const body = await response.text(); const body = await response.text();
apps/web/src/pages/AdminPage.tsx
+13 -9
View File
@@ -26,6 +26,7 @@ import type { AdminSection, LoadState, PlatformWithModelsInput } from '../types'
import { AccessRulesPanel } from './admin/AccessRulesPanel'; import { AccessRulesPanel } from './admin/AccessRulesPanel';
import { AuditLogsPanel } from './admin/AuditLogsPanel'; import { AuditLogsPanel } from './admin/AuditLogsPanel';
import { BaseModelCatalogPanel } from './admin/BaseModelCatalogPanel'; import { BaseModelCatalogPanel } from './admin/BaseModelCatalogPanel';
import { BillingSettlementsPanel } from './admin/BillingSettlementsPanel';
import { TenantsPanel, UserGroupsPanel, UsersPanel } from './admin/IdentityManagementPanels'; import { TenantsPanel, UserGroupsPanel, UsersPanel } from './admin/IdentityManagementPanels';
import { PlatformManagementPanel } from './admin/PlatformManagementPanel'; import { PlatformManagementPanel } from './admin/PlatformManagementPanel';
import { PricingRulesPanel } from './admin/PricingRulesPanel'; import { PricingRulesPanel } from './admin/PricingRulesPanel';
@@ -131,15 +132,18 @@ export function AdminPage(props: {
/> />
)} )}
{props.section === 'runtime' && ( {props.section === 'runtime' && (
<RuntimePoliciesPanel <div className="pageStack">
message={props.operationMessage} <BillingSettlementsPanel token={props.token} />
runnerPolicy={props.data.runnerPolicy} <RuntimePoliciesPanel
runtimePolicySets={props.data.runtimePolicySets} message={props.operationMessage}
state={props.state} runnerPolicy={props.data.runnerPolicy}
onDeleteRuntimePolicySet={props.onDeleteRuntimePolicySet} runtimePolicySets={props.data.runtimePolicySets}
onSaveRunnerPolicy={props.onSaveRunnerPolicy} state={props.state}
onSaveRuntimePolicySet={props.onSaveRuntimePolicySet} onDeleteRuntimePolicySet={props.onDeleteRuntimePolicySet}
/> onSaveRunnerPolicy={props.onSaveRunnerPolicy}
onSaveRuntimePolicySet={props.onSaveRuntimePolicySet}
/>
</div>
)} )}
{props.section === 'accessRules' && ( {props.section === 'accessRules' && (
<AccessRulesPanel <AccessRulesPanel
apps/web/src/pages/PlaygroundPage.pricing.test.ts
+57
View File
@@ -0,0 +1,57 @@
import { describe, expect, it } from 'vitest';
import type { GatewayPricingEstimate } from '@easyai-ai-gateway/contracts';
import {
MEDIA_ESTIMATE_DEBOUNCE_MS,
billingEstimateSignature,
mediaEstimateAllowsProduction,
mediaEstimateFromResponse,
} from './PlaygroundPage';
describe('playground pricing estimate', () => {
it('uses the specified 350ms debounce interval', () => {
expect(MEDIA_ESTIMATE_DEBOUNCE_MS).toBe(350);
});
it('only changes its signature for billing-relevant parameters', () => {
const first = billingEstimateSignature({
kind: 'images.generations', model: 'image-model', prompt: 'first',
count: 1, quality: 'high', resolution: '2K',
});
const promptOnly = billingEstimateSignature({
kind: 'images.generations', model: 'image-model', prompt: 'second',
count: 1, quality: 'high', resolution: '2K',
});
const changedCount = billingEstimateSignature({
kind: 'images.generations', model: 'image-model', prompt: 'second',
count: 2, quality: 'high', resolution: '2K',
});
expect(promptOnly).toBe(first);
expect(changedCount).not.toBe(first);
});
it('distinguishes explicit free and exposes the reservation ceiling', () => {
const response: GatewayPricingEstimate = {
candidateCount: 2,
currency: 'resource',
items: [{ amount: 0, currency: 'resource', isFree: true }],
pricingVersion: 'effective-pricing-v2',
requestFingerprint: 'fingerprint',
reservationAmount: 0,
resolver: 'effective-pricing-v2',
totalAmount: 0,
};
expect(mediaEstimateFromResponse(response)).toMatchObject({
amount: 0,
candidateCount: 2,
reservationAmount: 0,
status: 'free',
});
});
it('only allows production submission for the current pricing signature', () => {
expect(mediaEstimateAllowsProduction({ status: 'ready', signature: 'current' }, 'current')).toBe(true);
expect(mediaEstimateAllowsProduction({ status: 'free', signature: 'current' }, 'current')).toBe(true);
expect(mediaEstimateAllowsProduction({ status: 'ready', signature: 'stale' }, 'current')).toBe(false);
expect(mediaEstimateAllowsProduction({ status: 'loading', signature: 'current' }, 'current')).toBe(false);
});
});
apps/web/src/pages/PlaygroundPage.tsx
+123 -20
View File
@@ -2,7 +2,7 @@ import { useEffect, useMemo, useRef, useState } from 'react';
import type { GatewayApiKey, GatewayPricingEstimate, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts'; import type { GatewayApiKey, GatewayPricingEstimate, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts';
import { ArrowUp, ChevronDown, MessageSquarePlus, Settings2, Sparkles } from 'lucide-react'; import { ArrowUp, ChevronDown, MessageSquarePlus, Settings2, Sparkles } from 'lucide-react';
import { Badge, Button, FormDialog, Select, Textarea } from '../components/ui'; import { Badge, Button, FormDialog, Select, Textarea } from '../components/ui';
import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, estimatePricing, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api'; import { GatewayApiError, createImageEditTask, createImageGenerationTask, createVideoGenerationTask, estimatePricing, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api';
import type { PlaygroundMode } from '../types'; import type { PlaygroundMode } from '../types';
import { import {
PlaygroundPromptMentionInput, PlaygroundPromptMentionInput,
@@ -56,13 +56,18 @@ import {
const MEDIA_RUNS_STORAGE_KEY = 'easyai:playground:media-runs:v1'; const MEDIA_RUNS_STORAGE_KEY = 'easyai:playground:media-runs:v1';
const MEDIA_RUNS_STORAGE_LIMIT = 50; const MEDIA_RUNS_STORAGE_LIMIT = 50;
export const MEDIA_ESTIMATE_DEBOUNCE_MS = 350;
type MediaEstimateState = { type MediaEstimateState = {
amount?: number; amount?: number;
reservationAmount?: number;
candidateCount?: number;
currency?: string; currency?: string;
error?: string; error?: string;
pricingVersion?: string;
resolver?: string; resolver?: string;
status: 'idle' | 'loading' | 'ready' | 'error'; signature?: string;
status: 'idle' | 'loading' | 'ready' | 'free' | 'unavailable' | 'error';
}; };
const publicWorks = [ const publicWorks = [
@@ -107,6 +112,7 @@ export function PlaygroundPage(props: {
const [settingsOpen, setSettingsOpen] = useState(false); const [settingsOpen, setSettingsOpen] = useState(false);
const isMountedRef = useRef(false); const isMountedRef = useRef(false);
const pendingMediaModelRef = useRef(''); const pendingMediaModelRef = useRef('');
const mediaEstimateSequenceRef = useRef(0);
const resumedTaskIdsRef = useRef(new Set<string>()); const resumedTaskIdsRef = useRef(new Set<string>());
const activeMode = useMemo(() => modeOptions.find((item) => item.value === props.mode) ?? modeOptions[0], [props.mode]); const activeMode = useMemo(() => modeOptions.find((item) => item.value === props.mode) ?? modeOptions[0], [props.mode]);
const mediaUploadAcceptValue = sharedMediaUploadAcceptForMode(props.mode, videoMode); const mediaUploadAcceptValue = sharedMediaUploadAcceptForMode(props.mode, videoMode);
@@ -136,6 +142,10 @@ export function PlaygroundPage(props: {
supportsQualityControl: mediaCapabilities?.supportsQualityControl, supportsQualityControl: mediaCapabilities?.supportsQualityControl,
}); });
}, [mediaCapabilities, mediaSettings, mediaUploads, prompt, props.mode, selectedModel, videoMode]); }, [mediaCapabilities, mediaSettings, mediaUploads, prompt, props.mode, selectedModel, videoMode]);
const mediaEstimateSignature = useMemo(
() => billingEstimateSignature(mediaEstimatePayload),
[mediaEstimatePayload],
);
useEffect(() => { useEffect(() => {
setSelectedModel((current) => { setSelectedModel((current) => {
@@ -176,30 +186,35 @@ export function PlaygroundPage(props: {
useEffect(() => { useEffect(() => {
const credential = activeApiKeySecret || props.token; const credential = activeApiKeySecret || props.token;
if (props.mode === 'chat' || !credential || !mediaEstimatePayload) { if (props.mode === 'chat' || !credential || !mediaEstimatePayload) {
mediaEstimateSequenceRef.current += 1;
setMediaEstimate({ status: 'idle' }); setMediaEstimate({ status: 'idle' });
return; return;
} }
let cancelled = false; const sequence = mediaEstimateSequenceRef.current + 1;
setMediaEstimate((current) => ({ ...current, error: '', status: 'loading' })); mediaEstimateSequenceRef.current = sequence;
const controller = new AbortController();
setMediaEstimate((current) => ({ ...current, error: '', signature: mediaEstimateSignature, status: 'loading' }));
const timer = window.setTimeout(() => { const timer = window.setTimeout(() => {
estimatePricing(credential, mediaEstimatePayload) estimatePricing(credential, mediaEstimatePayload, controller.signal)
.then((estimate) => { .then((estimate) => {
if (cancelled) return; if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return;
setMediaEstimate(mediaEstimateFromResponse(estimate)); setMediaEstimate({ ...mediaEstimateFromResponse(estimate), signature: mediaEstimateSignature });
}) })
.catch((err) => { .catch((err) => {
if (cancelled) return; if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return;
const unavailable = err instanceof GatewayApiError && err.details.code === 'pricing_unavailable';
setMediaEstimate({ setMediaEstimate({
error: err instanceof Error ? err.message : '预计扣费计算失败', error: err instanceof Error ? err.message : unavailable ? '价格不可用' : '预计扣费计算失败',
status: 'error', signature: mediaEstimateSignature,
status: unavailable ? 'unavailable' : 'error',
}); });
}); });
}, 260); }, MEDIA_ESTIMATE_DEBOUNCE_MS);
return () => { return () => {
cancelled = true;
window.clearTimeout(timer); window.clearTimeout(timer);
controller.abort();
}; };
}, [activeApiKeySecret, mediaEstimatePayload, props.mode, props.token]); }, [activeApiKeySecret, mediaEstimateSignature, props.mode, props.token]);
useEffect(() => { useEffect(() => {
if (props.mode === 'image') { if (props.mode === 'image') {
@@ -313,11 +328,34 @@ export function PlaygroundPage(props: {
setMediaMessage(runMode === 'video' ? '请输入视频提示词。' : '请输入图片提示词。'); setMediaMessage(runMode === 'video' ? '请输入视频提示词。' : '请输入图片提示词。');
return; return;
} }
const localId = newLocalId();
const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads); const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads);
const runVideoMode = overrides?.videoMode ?? videoMode; const runVideoMode = overrides?.videoMode ?? videoMode;
const runModelOption = modelOptions.find((item) => item.value === runModel); const runModelOption = modelOptions.find((item) => item.value === runModel);
const runEstimatePayload = buildMediaEstimatePayload(runMode, runModel, trimmedPrompt, runSettings, runUploads, runVideoMode, {
supportsQualityControl: runModelOption
? deriveMediaModelCapabilities(runModelOption.models, runMode, runVideoMode, runSettings.resolution).supportsQualityControl
: mediaCapabilities?.supportsQualityControl,
});
const runEstimateSignature = billingEstimateSignature(runEstimatePayload);
if (!overrides && !mediaEstimateAllowsProduction(mediaEstimate, runEstimateSignature)) {
setMediaMessage(mediaEstimate.status === 'unavailable'
? '当前参数没有可用价格，已阻止生产生成。'
: '请等待当前参数的预计费用计算完成后再生成。');
return;
}
if (overrides) {
try {
await estimatePricing(credential, runEstimatePayload);
} catch (error) {
const unavailable = error instanceof GatewayApiError && error.details.code === 'pricing_unavailable';
setMediaMessage(unavailable
? '当前参数没有可用价格，已阻止生产生成。'
: `预计扣费计算失败，未提交生产任务：${error instanceof Error ? error.message : '未知错误'}`);
return;
}
}
const localId = newLocalId();
const modelLabel = runModelOption?.label ?? runModel; const modelLabel = runModelOption?.label ?? runModel;
const run: MediaGenerationRun = { const run: MediaGenerationRun = {
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
@@ -456,6 +494,7 @@ export function PlaygroundPage(props: {
imageHasReference={effectiveImageHasReference} imageHasReference={effectiveImageHasReference}
mediaSettings={mediaSettings} mediaSettings={mediaSettings}
mediaEstimate={mediaEstimate} mediaEstimate={mediaEstimate}
submitDisabled={!mediaEstimateAllowsProduction(mediaEstimate, mediaEstimateSignature)}
mediaCapabilities={mediaCapabilities} mediaCapabilities={mediaCapabilities}
uploadAccept={mediaUploadAcceptValue} uploadAccept={mediaUploadAcceptValue}
uploadMessage={mediaUploadMessage} uploadMessage={mediaUploadMessage}
@@ -684,6 +723,7 @@ function Composer(props: {
imageHasReference?: boolean; imageHasReference?: boolean;
mediaCapabilities?: MediaModelCapabilities; mediaCapabilities?: MediaModelCapabilities;
mediaEstimate?: MediaEstimateState; mediaEstimate?: MediaEstimateState;
submitDisabled?: boolean;
mediaSettings?: MediaGenerationSettings; mediaSettings?: MediaGenerationSettings;
mode: PlaygroundMode; mode: PlaygroundMode;
modelOptions: ModelOption[]; modelOptions: ModelOption[];
@@ -791,7 +831,7 @@ function Composer(props: {
<span>{mediaEstimateText(props.mediaEstimate)}</span> <span>{mediaEstimateText(props.mediaEstimate)}</span>
</span> </span>
)} )}
<Button type="button" size="icon" className="composerMediaSendButton" aria-label="发送测试" onClick={props.onSubmit}> <Button type="button" size="icon" className="composerMediaSendButton" aria-label="发送测试" disabled={props.submitDisabled} onClick={props.onSubmit}>
<ArrowUp size={24} /> <ArrowUp size={24} />
</Button> </Button>
</div> </div>
@@ -828,12 +868,17 @@ function buildMediaEstimatePayload(
}; };
} }
function mediaEstimateFromResponse(response: GatewayPricingEstimate): MediaEstimateState { export function mediaEstimateFromResponse(response: GatewayPricingEstimate): MediaEstimateState {
const amount = numericFromUnknown(response.totalAmount) ?? estimateItemsTotal(response.items);
const explicitFree = amount === 0 && response.items.length > 0 && response.items.every((item) => item.isFree === true);
return { return {
amount: numericFromUnknown(response.totalAmount) ?? estimateItemsTotal(response.items), amount,
reservationAmount: numericFromUnknown(response.reservationAmount),
candidateCount: response.candidateCount,
currency: response.currency || estimateItemsCurrency(response.items), currency: response.currency || estimateItemsCurrency(response.items),
pricingVersion: response.pricingVersion,
resolver: response.resolver, resolver: response.resolver,
status: 'ready', status: explicitFree ? 'free' : 'ready',
}; };
} }
@@ -847,21 +892,79 @@ function estimateItemsCurrency(items: GatewayPricingEstimate['items']) {
} }
function mediaEstimateText(estimate: MediaEstimateState) { function mediaEstimateText(estimate: MediaEstimateState) {
if (estimate.status === 'loading') return '计算中';
if (estimate.status === 'unavailable') return '价格不可用';
if (estimate.status === 'error') return '估价失败';
if (estimate.status === 'free') return '免费';
if (estimate.amount === undefined) return '--'; if (estimate.amount === undefined) return '--';
return formatEstimateAmount(estimate.amount); const amount = formatEstimateAmount(estimate.amount);
const reservation = estimate.reservationAmount;
if (reservation !== undefined && reservation > estimate.amount) {
return `预计 ${amount} · 冻结上限 ${formatEstimateAmount(reservation)}`;
}
return `预计 ${amount}`;
} }
function mediaEstimateHint(estimate: MediaEstimateState) { function mediaEstimateHint(estimate: MediaEstimateState) {
if (estimate.status === 'unavailable' && estimate.error) return `价格不可用：${estimate.error}`;
if (estimate.status === 'error' && estimate.error) return `预计扣费计算失败：${estimate.error}`; if (estimate.status === 'error' && estimate.error) return `预计扣费计算失败：${estimate.error}`;
if (estimate.status === 'free') return '当前计价规则已显式标记为免费';
return '扣费为预计，实际扣费以账单为准'; return '扣费为预计，实际扣费以账单为准';
} }
function mediaEstimateAriaLabel(estimate: MediaEstimateState) { function mediaEstimateAriaLabel(estimate: MediaEstimateState) {
if (estimate.status === 'error') return estimate.error ? `预计扣费计算失败：${estimate.error}` : '预计扣费计算失败'; if (estimate.status === 'error') return estimate.error ? `预计扣费计算失败：${estimate.error}` : '预计扣费计算失败';
if (estimate.status === 'unavailable') return estimate.error ? `价格不可用：${estimate.error}` : '价格不可用';
if (estimate.status === 'free') return '当前请求免费';
if (estimate.amount === undefined) return '预计扣费估算中'; if (estimate.amount === undefined) return '预计扣费估算中';
return `预计扣费 ${formatEstimateAmount(estimate.amount)} ${estimate.currency || 'resource'}`; return `预计扣费 ${formatEstimateAmount(estimate.amount)} ${estimate.currency || 'resource'}`;
} }
export function mediaEstimateAllowsProduction(estimate: MediaEstimateState, signature: string) {
return estimate.signature === signature && (estimate.status === 'ready' || estimate.status === 'free');
}
export function billingEstimateSignature(payload: Record<string, unknown> | null) {
if (!payload) return '';
const content = Array.isArray(payload.content)
? payload.content.map((item) => {
const record = item && typeof item === 'object' ? item as Record<string, unknown> : {};
return {
duration: record.duration,
role: record.role,
type: record.type,
hasAudio: Boolean(record.audio_url),
hasImage: Boolean(record.image_url),
hasVideo: Boolean(record.video_url),
};
})
: undefined;
const signature = {
kind: payload.kind,
model: payload.model,
n: payload.n ?? payload.count ?? payload.batch_size ?? payload.batchSize,
quality: payload.quality,
size: payload.size,
resolution: payload.resolution,
duration: payload.duration,
audio: payload.audio,
maxCompletionTokens: payload.max_completion_tokens,
maxOutputTokens: payload.max_output_tokens,
maxTokens: payload.max_tokens,
content,
imageCount: pricingReferenceCount(payload, ['image', 'images', 'image_url', 'image_urls', 'referenceImage', 'reference_image']),
};
return JSON.stringify(signature);
}
function pricingReferenceCount(payload: Record<string, unknown>, keys: string[]) {
return keys.reduce((count, key) => {
const value = payload[key];
if (Array.isArray(value)) return count + value.length;
return count + (value ? 1 : 0);
}, 0);
}
function formatEstimateAmount(value: number) { function formatEstimateAmount(value: number) {
if (!Number.isFinite(value)) return '--'; if (!Number.isFinite(value)) return '--';
const digits = Math.abs(value) > 0 && Math.abs(value) < 0.01 ? 6 : 2; const digits = Math.abs(value) > 0 && Math.abs(value) < 0.01 ? 6 : 2;
apps/web/src/pages/admin/BillingSettlementsPanel.tsx
+162
View File
@@ -0,0 +1,162 @@
import { useCallback, useEffect, useState } from 'react';
import { AlertTriangle, ReceiptText, RefreshCw, RotateCcw } from 'lucide-react';
import type { BillingSettlement } from '@easyai-ai-gateway/contracts';
import { listBillingSettlements, retryBillingSettlement } from '../../api';
import { Badge, Button, Card, CardContent, CardHeader, CardTitle, ConfirmDialog, Select, Table, TableCell, TableHead, TableRow } from '../../components/ui';
export function BillingSettlementsPanel(props: { token: string }) {
const [items, setItems] = useState<BillingSettlement[]>([]);
const [status, setStatus] = useState('');
const [loading, setLoading] = useState(false);
const [retrying, setRetrying] = useState('');
const [confirming, setConfirming] = useState<BillingSettlement | null>(null);
const [message, setMessage] = useState('');
const load = useCallback(async (signal?: AbortSignal) => {
setLoading(true);
setMessage('');
try {
const result = await listBillingSettlements(props.token, { status, pageSize: 100, signal });
setItems(result.items);
} catch (error) {
if (signal?.aborted) return;
setMessage(error instanceof Error ? error.message : '结算队列加载失败');
} finally {
if (!signal?.aborted) setLoading(false);
}
}, [props.token, status]);
useEffect(() => {
const controller = new AbortController();
void load(controller.signal);
return () => controller.abort();
}, [load]);
async function retry(item: BillingSettlement) {
setRetrying(item.id);
setMessage('');
try {
await retryBillingSettlement(props.token, item.id);
setMessage(`结算 ${shortId(item.id)} 已重新进入队列`);
await load();
} catch (error) {
setMessage(error instanceof Error ? error.message : '结算重试失败');
} finally {
setRetrying('');
}
}
const exceptions = items.filter((item) => item.status === 'retryable_failed' || item.status === 'manual_review').length;
return (
<div className="pageStack">
<Card>
<CardHeader>
<div className="identityHeaderTitle">
<div className="iconBox"><ReceiptText size={17} /></div>
<div>
<CardTitle>计费结算</CardTitle>
<p className="mutedText">任务结果与账务状态相互独立；异常记录可审计重试，不会重新调用上游。</p>
</div>
<Badge variant={exceptions ? 'warning' : 'secondary'}>{exceptions ? `${exceptions} 条异常` : `${items.length} 条`}</Badge>
</div>
</CardHeader>
<CardContent>
<div className="tableActions">
<Select size="sm" aria-label="结算状态" value={status} onChange={(event) => setStatus(event.target.value)}>
<option value="">全部状态</option>
<option value="pending">待处理</option>
<option value="processing">处理中</option>
<option value="retryable_failed">等待重试</option>
<option value="manual_review">人工复核</option>
<option value="completed">已完成</option>
</Select>
<Button type="button" size="sm" variant="outline" disabled={loading} onClick={() => void load()}>
<RefreshCw size={14} />{loading ? '刷新中' : '刷新'}
</Button>
</div>
{message && <p className="formMessage">{message}</p>}
</CardContent>
</Card>
{items.length ? (
<Table density="compact" className="identityDataTable">
<TableRow>
<TableHead>任务 / 动作</TableHead>
<TableHead>状态</TableHead>
<TableHead>金额</TableHead>
<TableHead>尝试</TableHead>
<TableHead>异常分类</TableHead>
<TableHead>操作</TableHead>
</TableRow>
{items.map((item) => (
<TableRow key={item.id}>
<TableCell><span className="identityTableName"><strong>{shortId(item.taskId)}</strong><small>{item.action === 'settle' ? '扣费' : '释放冻结'} · {shortId(item.id)}</small></span></TableCell>
<TableCell><Badge variant={statusVariant(item.status)}>{statusLabel(item.status)}</Badge></TableCell>
<TableCell>{formatAmount(item.amount)} {item.currency}</TableCell>
<TableCell>{item.attempts} / 20</TableCell>
<TableCell>{item.lastErrorCode || item.manualReviewReason || '-'}</TableCell>
<TableCell>
{(item.status === 'retryable_failed' || item.status === 'manual_review') ? (
<Button type="button" size="xs" variant="outline" disabled={retrying === item.id} onClick={() => setConfirming(item)}>
<RotateCcw size={13} />{retrying === item.id ? '提交中' : retryActionLabel(item)}
</Button>
) : '-'}
</TableCell>
</TableRow>
))}
</Table>
) : (
<Card><CardContent className="emptyState"><AlertTriangle size={18} /><strong>{loading ? '正在加载结算队列' : '暂无结算记录'}</strong><span>可切换状态筛选，查看待处理与人工复核记录。</span></CardContent></Card>
)}
<ConfirmDialog
open={Boolean(confirming)}
loading={Boolean(confirming && retrying === confirming.id)}
title={confirming ? `${retryActionLabel(confirming)}这条计费记录？` : '处理计费记录？'}
description={confirming ? retryActionDescription(confirming) : ''}
confirmLabel={confirming ? retryActionLabel(confirming) : '确认'}
onCancel={() => setConfirming(null)}
onConfirm={() => {
if (!confirming) return;
const item = confirming;
void retry(item).finally(() => setConfirming(null));
}}
/>
</div>
);
}
function statusLabel(status: string) {
return ({ pending: '待处理', processing: '处理中', retryable_failed: '等待重试', manual_review: '人工复核', completed: '已完成' } as Record<string, string>)[status] ?? status;
}
function statusVariant(status: string): 'secondary' | 'warning' | 'destructive' | 'success' | 'outline' {
if (status === 'completed') return 'success';
if (status === 'manual_review') return 'destructive';
if (status === 'retryable_failed') return 'warning';
if (status === 'processing') return 'outline';
return 'secondary';
}
function formatAmount(value: number) {
return new Intl.NumberFormat('zh-CN', { minimumFractionDigits: 0, maximumFractionDigits: 9 }).format(value);
}
function retryActionLabel(item: BillingSettlement) {
if (item.status !== 'manual_review') return '重试';
if (item.manualReviewReason === 'upstream_submission_unknown' && item.action === 'release') return '确认释放';
if (item.manualReviewReason === 'historical_success_without_charge' && item.action === 'settle') return '确认结算';
return '重新入队';
}
function retryActionDescription(item: BillingSettlement) {
if (item.manualReviewReason === 'upstream_submission_unknown' && item.action === 'release') {
return '上游提交结果不明。确认后只会释放当前冻结，不会再次调用上游；请先完成外部核查。';
}
if (item.manualReviewReason === 'historical_success_without_charge' && item.action === 'settle') {
return `确认后将按历史快照结算 ${formatAmount(item.amount)} ${item.currency}，不会重新调用上游。`;
}
return '确认后会将该记录重新放入结算队列，不会重新调用上游。';
}
function shortId(value: string) {
return value.length > 12 ? `${value.slice(0, 8)}…${value.slice(-4)}` : value;
}
apps/web/src/pages/admin/PricingRuleVisualEditor.tsx
+37 -4
View File
@@ -221,7 +221,17 @@ function ModeRuleEditor(props: {
<Select size="sm" value={rule.status ?? 'active'} onChange={(event) => patch({ status: event.target.value })}>{statuses.map((item) => <option key={item} value={item}>{item}</option>)}</Select> <Select size="sm" value={rule.status ?? 'active'} onChange={(event) => patch({ status: event.target.value })}>{statuses.map((item) => <option key={item} value={item}>{item}</option>)}</Select>
</PricingFormRow> </PricingFormRow>
<PricingFormRow label={`基础单价/${unitLabel(rule.unit)}`}> <PricingFormRow label={`基础单价/${unitLabel(rule.unit)}`}>
<Input size="sm" min="0" step="0.0001" type="number" value={rule.basePrice} onChange={(event) => patch({ basePrice: Number(event.target.value) })} /> <Input disabled={rule.isFree} size="sm" min="0" step="0.000000001" type="number" value={rule.basePrice} onChange={(event) => patch({ basePrice: Number(event.target.value) })} />
</PricingFormRow>
<PricingFormRow label="显式免费">
<label>
<input
type="checkbox"
checked={rule.isFree ?? false}
onChange={(event) => patch({ isFree: event.target.checked, basePrice: event.target.checked ? 0 : rule.basePrice })}
/>
<span>允许该规则以零费用运行</span>
</label>
</PricingFormRow> </PricingFormRow>
<PricingReadonlyRow label="计价单位" value={unitLabel(rule.unit)} /> <PricingReadonlyRow label="计价单位" value={unitLabel(rule.unit)} />
<PricingReadonlyRow label="计算方式" value={calculatorLabel(rule.calculatorType)} /> <PricingReadonlyRow label="计算方式" value={calculatorLabel(rule.calculatorType)} />
@@ -311,11 +321,32 @@ function TextRuleEditor(props: {
<PricingFormRow label="状态"> <PricingFormRow label="状态">
<Select size="sm" value={props.rule.status ?? 'active'} onChange={(event) => props.onChange({ ...props.rule, status: event.target.value })}>{statuses.map((item) => <option key={item} value={item}>{item}</option>)}</Select> <Select size="sm" value={props.rule.status ?? 'active'} onChange={(event) => props.onChange({ ...props.rule, status: event.target.value })}>{statuses.map((item) => <option key={item} value={item}>{item}</option>)}</Select>
</PricingFormRow> </PricingFormRow>
<PricingFormRow label="显式免费">
<label>
<input
type="checkbox"
checked={props.rule.isFree ?? false}
onChange={(event) => {
const isFree = event.target.checked;
props.onChange({
...props.rule,
isFree,
basePrice: isFree ? 0 : props.rule.basePrice,
formulaConfig: isFree
? { ...(props.rule.formulaConfig ?? {}), inputTokenPrice: 0, cachedInputTokenPrice: 0, outputTokenPrice: 0 }
: props.rule.formulaConfig,
});
}}
/>
<span>允许该规则以零费用运行</span>
</label>
</PricingFormRow>
<PricingFormRow label="输入单价/1K tokens"> <PricingFormRow label="输入单价/1K tokens">
<Input <Input
disabled={props.rule.isFree}
size="sm" size="sm"
min="0" min="0"
step="0.0001" step="0.000000001"
type="number" type="number"
value={prices.inputTokenPrice} value={prices.inputTokenPrice}
onChange={(event) => onChange={(event) =>
@@ -329,9 +360,10 @@ function TextRuleEditor(props: {
</PricingFormRow> </PricingFormRow>
<PricingFormRow label="缓存输入单价/1K tokens"> <PricingFormRow label="缓存输入单价/1K tokens">
<Input <Input
disabled={props.rule.isFree}
size="sm" size="sm"
min="0" min="0"
step="0.0001" step="0.000000001"
type="number" type="number"
value={prices.cachedInputTokenPrice} value={prices.cachedInputTokenPrice}
onChange={(event) => onChange={(event) =>
@@ -345,9 +377,10 @@ function TextRuleEditor(props: {
</PricingFormRow> </PricingFormRow>
<PricingFormRow label="输出单价/1K tokens"> <PricingFormRow label="输出单价/1K tokens">
<Input <Input
disabled={props.rule.isFree}
size="sm" size="sm"
min="0" min="0"
step="0.0001" step="0.000000001"
type="number" type="number"
value={prices.outputTokenPrice} value={prices.outputTokenPrice}
onChange={(event) => onChange={(event) =>
apps/web/src/pages/admin/PricingRulesPanel.tsx
+3
View File
@@ -225,6 +225,7 @@ function ruleToInput(rule: PricingRule): PricingRuleInput {
resourceType: rule.resourceType, resourceType: rule.resourceType,
unit: rule.unit, unit: rule.unit,
basePrice: rule.basePrice, basePrice: rule.basePrice,
isFree: rule.isFree,
currency: rule.currency, currency: rule.currency,
baseWeight: rule.baseWeight ?? {}, baseWeight: rule.baseWeight ?? {},
dynamicWeight: rule.dynamicWeight ?? {}, dynamicWeight: rule.dynamicWeight ?? {},
@@ -234,6 +235,8 @@ function ruleToInput(rule: PricingRule): PricingRuleInput {
priority: rule.priority, priority: rule.priority,
status: rule.status, status: rule.status,
metadata: rule.metadata ?? {}, metadata: rule.metadata ?? {},
effectiveFrom: rule.effectiveFrom,
effectiveTo: rule.effectiveTo,
}; };
} }
docs/billing-flow-v2.md
+65
View File
@@ -0,0 +1,65 @@
# 计费业务流程 v2
## 正常流程
```mermaid
sequenceDiagram
participant UI as 前端
participant API as Gateway API
participant DB as PostgreSQL
participant UP as 上游模型
participant BW as 结算 Worker
UI->>API: 参数变化后的只读估价
API->>DB: 读取有效计价规则
API-->>UI: 首选预计费用 + 候选最大冻结额
UI->>API: 正式提交（可带 Idempotency-Key）
API->>DB: 重新计价、保存快照并冻结最大金额
API->>DB: 领取执行租约
API->>UP: 提交已预处理请求
UP-->>API: 成功结果与实际用量
API->>DB: 原子写 Attempt、任务成功、最终金额和 settle Outbox
API-->>UI: succeeded + billingStatus=pending
BW->>DB: SKIP LOCKED 领取 Outbox
BW->>DB: 原子扣费、释放冻结、完成 Outbox
```
## 失败与取消
上游提交前失败、上游明确失败和用户取消均在任务终态事务内写入 `release` Outbox。Worker 使用冻结流水的幂等键释放资金。若上游请求已经进入 `submitting`，或成功响应已返回但任务终态尚未提交，租约接管必须停止自动执行；任务和对应 Outbox 转人工复核并保留冻结。
人工复核记录在管理端明确显示后续动作。上游结果不明的记录只能由 Manager 在完成外部核查后确认释放；历史成功未扣费且金额快照可用的记录可由 Manager 明确确认结算。两类操作均要求确认、`Idempotency-Key` 和审计记录，且不会再次调用上游。
## 即时估价
前端仅对影响计费的字段生成稳定签名，包括模型、生成模式、数量、质量、分辨率、时长、音频、参考素材和 Token 上限。签名变化后等待 350ms，取消旧请求，并用递增序号阻止乱序响应覆盖。
界面必须区分：
- 计算中；
- 显式免费；
- 正常预计费用与冻结上限；
- 价格不可用；
- 估价请求失败。
价格不可用或估价失败时禁止生产生成；模拟模式仍允许执行并显示原因。
## Worker 参数
- 轮询间隔：1 秒。
- 单批数量：50。
- 处理锁过期：120 秒。
- 最大自动尝试：20 次。
- 退避上限：15 分钟。
这些参数是安全默认值，可通过受控配置调整，但不得改变幂等键、快照和账务不变量。
## 发布与回滚
`BILLING_ENGINE_MODE` 支持：
- `observe`：并行计算并记录差异，不改变旧路径账务结果。
- `enforce`：启用 v2 缺价拒绝、冻结与结算闭环。
- `hold`：在调用上游前拒绝新的生产任务，已有 Outbox 继续处理。
发布先运行 24 小时 observe，补齐所有缺价和非法规则，再以 10% 流量启用 enforce，确认没有重复扣费、长期冻结和异常差额后扩大到 100%。回滚只切换到 hold，不回滚数据库增量，也不恢复隐式零价。
docs/decisions/002-billing-correctness-v2.md
+86
View File
@@ -0,0 +1,86 @@
# ADR-002：计费正确性闭环与即时估价
## 状态
Accepted
## 日期
2026-07-20
## 背景
Gateway 已具备 PostgreSQL 本地钱包、任务冻结、任务扣费和钱包流水，但任务终态与钱包结算分属不同事务。进程在任务成功写入后、扣费前或释放前退出时，可能留下成功未扣费或永久冻结。现有估价只计算首选候选，缺价会退化为普通零价，且文本输出上限和精度规则与正式结算并不完全一致。
Comfy 的业务流程证明了三项做法有参考价值：任务状态与计费状态分离；计费操作使用独立令牌和过期接管；参数变化后立即生成只读估价。Gateway 不采用 Comfy 的三十分钟虚拟预占、结算时刷新最新价格、公开估价接口和负余额策略，因为这些做法会削弱本地钱包作为唯一账务事实源的约束。
## 决策
### 账务事实源与状态
Gateway PostgreSQL 本地钱包是唯一账务事实源，不与 Comfy 或其他系统双写。任务状态与计费状态独立：生成成功可以处于 `succeeded + pending`，结算失败不得篡改生成结果。
计费状态为：
- `not_started`：尚未产生账务动作。
- `pending`：结算或释放已进入 Outbox。
- `processing`：Worker 已领取账务动作。
- `settled`：已幂等扣费并释放冻结。
- `released`：失败或取消任务的冻结已释放。
- `retryable_failed`：可自动重试。
- `manual_review`：上游结果不明或连续失败，需要人工处理。
- `not_required`：模拟任务、无本地钱包用户或其他明确无需计费的任务。
### 统一计价
估价、提交冻结和最终账单统一使用 `effective-pricing-v2`。规则优先级为平台模型、平台、基准模型；只接受处于生效期内、状态有效、币种为 `resource` 且计算器受支持的规则。普通零价、缺价或失效规则返回 `503 pricing_unavailable`；只有 `isFree=true` 的显式免费规则允许零费用。
金额决策使用 9 位十进制定点数。文本费用按实际 Token 比例计算，不按千 Token 向上取整；视频以五秒为一个单位向上取整。估价对所有有效候选计算并以最大值作为冻结上限，首选候选费用继续作为 `totalAmount`。
前端估价是只读提示，不构成价格承诺。正式提交必须重新计价、冻结，并保存不可变计价快照。
### 结算闭环
成功 Attempt、任务成功结果、最终金额、计价快照和 `settle` Outbox 在同一个数据库事务写入；失败和取消任务原子写入 `release` Outbox。Worker 使用 `FOR UPDATE SKIP LOCKED` 批量领取，锁超时后允许其他实例接管，并在钱包行锁事务中完成幂等扣费或释放、任务计费状态和 Outbox 完成状态。
上游提交状态分为 `not_submitted`、`submitting` 和 `response_received`。处于 `submitting` 且结果不明的任务保留冻结并进入人工复核，不自动重试上游。
### 请求幂等与执行租约
生成接口可选接受单值 `Idempotency-Key`。数据库只保存键摘要以及规范化请求摘要；同键同请求按协议重放，同键不同请求返回 `409 idempotency_key_reused`，流式重复返回 `409 idempotency_stream_replay_unsupported`。日志和审计记录不得包含原始键或完整请求正文。
任务由带随机 `execution_token` 的五分钟租约领取，每三十秒续约。运行态更新和终态提交必须匹配令牌，旧 Worker 不得覆盖接管后的新结果。
## 账务不变量
1. `balance >= frozen_balance >= 0`。
2. 同一任务和币种最多存在一笔有效扣费。
3. 一个冻结必须最终被结算消费或释放，不能同时发生两者。
4. 正式任务缺价时不得调用上游。
5. 最终扣费只能使用提交时保存的计价快照，不能读取更新后的价格。
6. 估价不得创建任务、冻结余额或写钱包流水。
7. 历史成功未扣费任务只标记人工复核，不自动追扣。
## 异常矩阵
| 发生点 | 任务状态 | 计费状态 | 自动动作 |
| --- | --- | --- | --- |
| 计价失败 | 未创建或失败 | `not_started` | 返回 `pricing_unavailable`，不调用上游 |
| 冻结失败 | 已创建 | `not_started` | 返回余额不足，不调用上游 |
| 上游提交前失败 | 失败 | `pending` | Outbox 释放冻结 |
| 上游提交结果不明 | 运行中或失败 | `manual_review` | 保留冻结，禁止自动重试上游 |
| 上游明确失败 | 失败 | `pending` | Outbox 释放冻结 |
| 生成成功、结算待执行 | 成功 | `pending` | 返回成功结果，Worker 异步结算 |
| 结算暂时失败 | 成功 | `retryable_failed` | 指数退避重试 |
| 结算连续失败 | 成功 | `manual_review` | 管理员审计化重试 |
## 历史数据策略
迁移只分类，不产生历史补扣：已有任务扣费流水的成功任务标记 `settled`；失败或取消任务的遗留冻结进入释放队列；成功但无扣费流水的生产任务进入 `manual_review`。
## 后果
- 成功结果和账务暂时失败可以独立呈现，调用方必须读取 `billingStatus`。
- 计价规则编辑器必须要求显式免费，并在规则不可用时阻止生产生成。
- 结算具有最终一致性，因此需要积压、延迟、重试和人工复核监控。
- 数据库增量在回滚时保留；应用切换到 `hold` 后拒绝新的生产任务，同时继续运行结算 Worker。
docs/runbooks/production-ci-cd.md
+9 -1
View File
@@ -16,7 +16,7 @@
生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。
provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。 provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。PostgreSQL 16 集成库作为固定 digest 的 Actions service container 运行，PR Job 不接收宿主或内层 Docker API。
## 首次安装或修复 CI Runner ## 首次安装或修复 CI Runner
@@ -34,6 +34,14 @@ RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh
./scripts/provision-ci-runner.sh ./scripts/provision-ci-runner.sh
``` ```
宿主无法直连 Docker Hub 时，可使用可信 HTTPS 镜像代理；Runner 来源仍必须携带脚本中完全相同的 manifest digest，内层 daemon 拉取的 Job 与 service 镜像也会继续校验工作流固定的 digest：
```bash
CI_RUNNER_IMAGE_SOURCE='docker.m.daocloud.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' \
CI_NESTED_DOCKER_REGISTRY_MIRROR='https://docker.m.daocloud.io' \
./scripts/provision-ci-runner.sh
```
注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。 注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。
默认磁盘门禁可用环境变量提高，但生产环境不应降低： 默认磁盘门禁可用环境变量提高，但生产环境不应降低：
packages/contracts/src/index.ts
+56
View File
@@ -184,6 +184,7 @@ export interface PricingRule {
| string; | string;
unit: '1k_tokens' | 'image' | '5s' | 'second' | 'character_1k' | 'item' | string; unit: '1k_tokens' | 'image' | '5s' | 'second' | 'character_1k' | 'item' | string;
basePrice: number; basePrice: number;
isFree: boolean;
currency: 'resource' | 'credit' | 'cny' | 'usd' | string; currency: 'resource' | 'credit' | 'cny' | 'usd' | string;
baseWeight?: Record<string, unknown>; baseWeight?: Record<string, unknown>;
dynamicWeight?: Record<string, unknown>; dynamicWeight?: Record<string, unknown>;
@@ -193,6 +194,8 @@ export interface PricingRule {
priority: number; priority: number;
status: 'active' | 'deprecated' | 'hidden' | string; status: 'active' | 'deprecated' | 'hidden' | string;
metadata?: Record<string, unknown>; metadata?: Record<string, unknown>;
effectiveFrom?: string;
effectiveTo?: string;
createdAt: string; createdAt: string;
updatedAt: string; updatedAt: string;
} }
@@ -203,6 +206,7 @@ export interface PricingRuleInput {
resourceType: string; resourceType: string;
unit: string; unit: string;
basePrice: number; basePrice: number;
isFree?: boolean;
currency?: string; currency?: string;
baseWeight?: Record<string, unknown>; baseWeight?: Record<string, unknown>;
dynamicWeight?: Record<string, unknown>; dynamicWeight?: Record<string, unknown>;
@@ -212,6 +216,8 @@ export interface PricingRuleInput {
priority?: number; priority?: number;
status?: 'active' | 'deprecated' | 'hidden' | string; status?: 'active' | 'deprecated' | 'hidden' | string;
metadata?: Record<string, unknown>; metadata?: Record<string, unknown>;
effectiveFrom?: string;
effectiveTo?: string;
} }
export interface PricingRuleSet { export interface PricingRuleSet {
@@ -548,7 +554,11 @@ export interface GatewayPricingEstimate {
items: GatewayPricingEstimateItem[]; items: GatewayPricingEstimateItem[];
resolver: string; resolver: string;
totalAmount?: number; totalAmount?: number;
reservationAmount?: number;
currency?: string; currency?: string;
candidateCount: number;
pricingVersion: string;
requestFingerprint: string;
} }
export interface GatewayWalletAccount { export interface GatewayWalletAccount {
@@ -1146,6 +1156,15 @@ export interface GatewayTask {
metrics?: Record<string, unknown>; metrics?: Record<string, unknown>;
billingSummary?: Record<string, unknown>; billingSummary?: Record<string, unknown>;
finalChargeAmount?: number; finalChargeAmount?: number;
billingVersion?: string;
billingStatus?: 'not_started' | 'pending' | 'processing' | 'settled' | 'released' | 'retryable_failed' | 'manual_review' | 'not_required' | string;
billingCurrency?: string;
pricingSnapshot?: Record<string, unknown>;
requestFingerprint?: string;
reservationAmount?: number;
executionLeaseExpiresAt?: string;
billingUpdatedAt?: string;
billingSettledAt?: string;
responseStartedAt?: string; responseStartedAt?: string;
responseFinishedAt?: string; responseFinishedAt?: string;
responseDurationMs?: number; responseDurationMs?: number;
@@ -1202,12 +1221,49 @@ export interface GatewayTaskAttempt {
responseStartedAt?: string; responseStartedAt?: string;
responseFinishedAt?: string; responseFinishedAt?: string;
responseDurationMs?: number; responseDurationMs?: number;
pricingSnapshot?: Record<string, unknown>;
requestFingerprint?: string;
upstreamSubmissionStatus?: 'not_submitted' | 'submitting' | 'response_received' | string;
upstreamSubmissionUpdatedAt?: string;
errorCode?: string; errorCode?: string;
errorMessage?: string; errorMessage?: string;
startedAt: string; startedAt: string;
finishedAt?: string; finishedAt?: string;
} }
export interface BillingSettlement {
id: string;
taskId: string;
action: 'settle' | 'release' | string;
amount: number;
currency: string;
status: 'pending' | 'processing' | 'retryable_failed' | 'completed' | 'manual_review' | string;
attempts: number;
nextAttemptAt: string;
lockedBy?: string;
lockedAt?: string;
lastErrorCode?: string;
lastErrorMessage?: string;
manualReviewReason?: string;
pricingSnapshot?: Record<string, unknown>;
payload?: Record<string, unknown>;
completedAt?: string;
createdAt: string;
updatedAt: string;
}
export interface BillingSettlementListResponse {
items: BillingSettlement[];
total: number;
page: number;
pageSize: number;
}
export interface BillingSettlementRetryResponse {
settlement: BillingSettlement;
auditLog?: GatewayAuditLog;
}
export interface GatewayTaskEvent { export interface GatewayTaskEvent {
id: string; id: string;
taskId: string; taskId: string;
pnpm-lock.yaml
Generated
+46 -19
View File
@@ -6,15 +6,17 @@ settings:
overrides: overrides:
'@babel/core@<=7.29.0': 7.29.6 '@babel/core@<=7.29.0': 7.29.6
dompurify@<=3.4.10: 3.4.11
esbuild@>=0.27.3 <0.28.1: 0.28.1
form-data@>=4.0.0 <4.0.6: 4.0.6
js-yaml@<3.15.0: 4.1.1
mermaid@>=11.0.0-alpha.1 <=11.14.0: 11.15.0
minimatch@>=9.0.0 <9.0.7: 9.0.7
'@nx/js>picomatch': 4.0.5 '@nx/js>picomatch': 4.0.5
'@nx/vite>picomatch': 4.0.5 '@nx/vite>picomatch': 4.0.5
'@nx/workspace>picomatch': 4.0.5 '@nx/workspace>picomatch': 4.0.5
axios@<1.18.0: 1.18.0
brace-expansion@>=2.0.0 <2.1.2: 2.1.2
dompurify@<=3.4.10: 3.4.11
esbuild@>=0.27.3 <0.28.1: 0.28.1
form-data@>=4.0.0 <4.0.6: 4.0.6
js-yaml@<4.3.0: 4.3.0
mermaid@>=11.0.0-alpha.1 <=11.14.0: 11.15.0
minimatch@>=9.0.0 <9.0.7: 9.0.7
tmp@<0.2.7: 0.2.7 tmp@<0.2.7: 0.2.7
importers: importers:
@@ -2443,6 +2445,10 @@ packages:
resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==} resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==}
engines: {node: '>= 10.0.0'} engines: {node: '>= 10.0.0'}
agent-base@6.0.2:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
engines: {node: '>= 6.0.0'}
ajv@8.20.0: ajv@8.20.0:
resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==}
@@ -2491,8 +2497,8 @@ packages:
asynckit@0.4.0: asynckit@0.4.0:
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
axios@1.16.0: axios@1.18.0:
resolution: {integrity: sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==} resolution: {integrity: sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==}
babel-plugin-const-enum@1.2.0: babel-plugin-const-enum@1.2.0:
resolution: {integrity: sha512-o1m/6iyyFnp9MRsK1dHF3bneqyf3AlM2q3A/YbgQr2pCat6B6XJVDv2TXqzfY2RYUi4mak6WAksSBPlyYGx9dg==} resolution: {integrity: sha512-o1m/6iyyFnp9MRsK1dHF3bneqyf3AlM2q3A/YbgQr2pCat6B6XJVDv2TXqzfY2RYUi4mak6WAksSBPlyYGx9dg==}
@@ -2553,8 +2559,8 @@ packages:
bl@4.1.0: bl@4.1.0:
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
brace-expansion@2.1.0: brace-expansion@2.1.2:
resolution: {integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==} resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==}
brace-expansion@5.0.7: brace-expansion@5.0.7:
resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==}
@@ -3184,6 +3190,10 @@ packages:
html-void-elements@3.0.0: html-void-elements@3.0.0:
resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
https-proxy-agent@5.0.1:
resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
engines: {node: '>= 6'}
iconv-lite@0.6.3: iconv-lite@0.6.3:
resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==}
engines: {node: '>=0.10.0'} engines: {node: '>=0.10.0'}
@@ -3281,8 +3291,8 @@ packages:
js-tokens@9.0.1: js-tokens@9.0.1:
resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==}
js-yaml@4.1.1: js-yaml@4.3.0:
resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==}
hasBin: true hasBin: true
jsesc@3.1.0: jsesc@3.1.0:
@@ -5820,6 +5830,7 @@ snapshots:
- '@swc-node/register' - '@swc-node/register'
- '@swc/core' - '@swc/core'
- debug - debug
- supports-color
'@phenomnomnominal/tsquery@5.0.1(typescript@5.9.3)': '@phenomnomnominal/tsquery@5.0.1(typescript@5.9.3)':
dependencies: dependencies:
@@ -7152,7 +7163,7 @@ snapshots:
'@yarnpkg/parsers@3.0.2': '@yarnpkg/parsers@3.0.2':
dependencies: dependencies:
js-yaml: 4.1.1 js-yaml: 4.3.0
tslib: 2.8.1 tslib: 2.8.1
'@zkochan/js-yaml@0.0.7': '@zkochan/js-yaml@0.0.7':
@@ -7161,6 +7172,12 @@ snapshots:
address@1.2.2: {} address@1.2.2: {}
agent-base@6.0.2:
dependencies:
debug: 4.4.3
transitivePeerDependencies:
- supports-color
ajv@8.20.0: ajv@8.20.0:
dependencies: dependencies:
fast-deep-equal: 3.1.3 fast-deep-equal: 3.1.3
@@ -7258,13 +7275,15 @@ snapshots:
asynckit@0.4.0: {} asynckit@0.4.0: {}
axios@1.16.0: axios@1.18.0:
dependencies: dependencies:
follow-redirects: 1.16.0 follow-redirects: 1.16.0
form-data: 4.0.6 form-data: 4.0.6
https-proxy-agent: 5.0.1
proxy-from-env: 2.1.0 proxy-from-env: 2.1.0
transitivePeerDependencies: transitivePeerDependencies:
- debug - debug
- supports-color
babel-plugin-const-enum@1.2.0(@babel/core@7.29.6): babel-plugin-const-enum@1.2.0(@babel/core@7.29.6):
dependencies: dependencies:
@@ -7336,7 +7355,7 @@ snapshots:
inherits: 2.0.4 inherits: 2.0.4
readable-stream: 3.6.2 readable-stream: 3.6.2
brace-expansion@2.1.0: brace-expansion@2.1.2:
dependencies: dependencies:
balanced-match: 1.0.2 balanced-match: 1.0.2
@@ -7863,7 +7882,7 @@ snapshots:
front-matter@4.0.2: front-matter@4.0.2:
dependencies: dependencies:
js-yaml: 4.1.1 js-yaml: 4.3.0
fs-constants@1.0.0: {} fs-constants@1.0.0: {}
@@ -8054,6 +8073,13 @@ snapshots:
html-void-elements@3.0.0: {} html-void-elements@3.0.0: {}
https-proxy-agent@5.0.1:
dependencies:
agent-base: 6.0.2
debug: 4.4.3
transitivePeerDependencies:
- supports-color
iconv-lite@0.6.3: iconv-lite@0.6.3:
dependencies: dependencies:
safer-buffer: 2.1.2 safer-buffer: 2.1.2
@@ -8129,7 +8155,7 @@ snapshots:
js-tokens@9.0.1: {} js-tokens@9.0.1: {}
js-yaml@4.1.1: js-yaml@4.3.0:
dependencies: dependencies:
argparse: 2.0.1 argparse: 2.0.1
@@ -8677,7 +8703,7 @@ snapshots:
minimatch@5.1.9: minimatch@5.1.9:
dependencies: dependencies:
brace-expansion: 2.1.0 brace-expansion: 2.1.2
minimatch@9.0.7: minimatch@9.0.7:
dependencies: dependencies:
@@ -8712,7 +8738,7 @@ snapshots:
'@yarnpkg/lockfile': 1.1.0 '@yarnpkg/lockfile': 1.1.0
'@yarnpkg/parsers': 3.0.2 '@yarnpkg/parsers': 3.0.2
'@zkochan/js-yaml': 0.0.7 '@zkochan/js-yaml': 0.0.7
axios: 1.16.0 axios: 1.18.0
chalk: 4.1.2 chalk: 4.1.2
cli-cursor: 3.1.0 cli-cursor: 3.1.0
cli-spinners: 2.6.1 cli-spinners: 2.6.1
@@ -8756,6 +8782,7 @@ snapshots:
'@nx/nx-win32-x64-msvc': 21.6.11 '@nx/nx-win32-x64-msvc': 21.6.11
transitivePeerDependencies: transitivePeerDependencies:
- debug - debug
- supports-color
once@1.4.0: once@1.4.0:
dependencies: dependencies:
pnpm-workspace.yaml
+3 -1
View File
@@ -8,10 +8,12 @@ allowBuilds:
overrides: overrides:
'@babel/core@<=7.29.0': 7.29.6 '@babel/core@<=7.29.0': 7.29.6
'axios@<1.18.0': 1.18.0
'brace-expansion@>=2.0.0 <2.1.2': 2.1.2
'dompurify@<=3.4.10': 3.4.11 'dompurify@<=3.4.10': 3.4.11
'esbuild@>=0.27.3 <0.28.1': 0.28.1 'esbuild@>=0.27.3 <0.28.1': 0.28.1
'form-data@>=4.0.0 <4.0.6': 4.0.6 'form-data@>=4.0.0 <4.0.6': 4.0.6
'js-yaml@<3.15.0': 4.1.1 'js-yaml@<4.3.0': 4.3.0
'mermaid@>=11.0.0-alpha.1 <=11.14.0': 11.15.0 'mermaid@>=11.0.0-alpha.1 <=11.14.0': 11.15.0
'minimatch@>=9.0.0 <9.0.7': 9.0.7 'minimatch@>=9.0.0 <9.0.7': 9.0.7
'@nx/js>picomatch': 4.0.5 '@nx/js>picomatch': 4.0.5
scripts/provision-ci-runner.sh
+3 -2
View File
@@ -188,14 +188,15 @@ tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREF
chmod 0755 "$PREFIX/bin/trivy" chmod 0755 "$PREFIX/bin/trivy"
GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \
"$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0
chown -R root:root "$PREFIX"
chmod -R go-w "$PREFIX"
require_free_space "$MIN_FREE_GIB" "runner image pull" require_free_space "$MIN_FREE_GIB" "runner image pull"
docker pull "$RUNNER_IMAGE_SOURCE" docker pull "$RUNNER_IMAGE_SOURCE"
runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE")
printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \
fail "pinned runner image did not resolve to an immutable local image ID" fail "pinned runner image did not resolve to an immutable local image ID"
chown -R root:root "$PREFIX"
chmod -R go-w "$PREFIX"
require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull" require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull"
docker volume create "$RUNNER_VOLUME" >/dev/null docker volume create "$RUNNER_VOLUME" >/dev/null
tests/ci/pipeline-test.sh
+5
View File
@@ -117,6 +117,11 @@ for quality_workflow in "$workflow" "$release_workflow"; do
grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow"
grep -Fq 'docker-compose version' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow"
grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow"
grep -Fq 'services:' "$quality_workflow"
grep -Fq 'image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' "$quality_workflow"
grep -Fq 'POSTGRES_HOST_AUTH_METHOD: trust' "$quality_workflow"
grep -Fq 'AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable' "$quality_workflow"
grep -Fq 'run: go run ./cmd/migrate' "$quality_workflow"
if grep -Fq 'docker compose' "$quality_workflow"; then if grep -Fq 'docker compose' "$quality_workflow"; then
echo 'job container must use the read-only standalone Compose binary' >&2 echo 'job container must use the read-only standalone Compose binary' >&2
exit 1 exit 1