From 01a013c809366c968b7afc5f511afaa230851832 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Mon, 20 Jul 2026 23:09:18 +0800 Subject: [PATCH 01/13] =?UTF-8?q?docs(billing):=20=E5=9B=BA=E5=8C=96?= =?UTF-8?q?=E8=AE=A1=E8=B4=B9=E9=97=AD=E7=8E=AF=E5=86=B3=E7=AD=96=E4=B8=8E?= =?UTF-8?q?=E8=BF=81=E7=A7=BB=E5=9F=BA=E7=A1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 ADR-002、计费流程说明和 0069 增量迁移，建立独立计费状态、结算 Outbox、显式免费与钱包约束。历史成功未扣费任务仅进入人工复核，不执行追扣。\n\n验证：迁移安全验证与 tests/ci/migrations-test.sh 通过。 --- .../0069_billing_correctness_v2.sql | 165 ++++++++++++++++++ docs/billing-flow-v2.md | 63 +++++++ docs/decisions/002-billing-correctness-v2.md | 86 +++++++++ 3 files changed, 314 insertions(+) create mode 100644 apps/api/migrations/0069_billing_correctness_v2.sql create mode 100644 docs/billing-flow-v2.md create mode 100644 docs/decisions/002-billing-correctness-v2.md diff --git a/apps/api/migrations/0069_billing_correctness_v2.sql b/apps/api/migrations/0069_billing_correctness_v2.sql new file mode 100644 index 0000000..87b0f9e --- /dev/null +++ b/apps/api/migrations/0069_billing_correctness_v2.sql @@ -0,0 +1,165 @@ +ALTER TABLE gateway_tasks + ADD COLUMN IF NOT EXISTS billing_version text NOT NULL DEFAULT 'effective-pricing-v2', + ADD COLUMN IF NOT EXISTS billing_status text NOT NULL DEFAULT 'not_started', + ADD COLUMN IF NOT EXISTS billing_currency text NOT NULL DEFAULT 'resource', + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS request_fingerprint text, + ADD COLUMN IF NOT EXISTS idempotency_key_hash text, + ADD COLUMN IF NOT EXISTS reservation_amount numeric(38, 9) NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS execution_token uuid, + ADD COLUMN IF NOT EXISTS execution_lease_expires_at timestamptz, + ADD COLUMN IF NOT EXISTS billing_updated_at timestamptz, + ADD COLUMN IF NOT EXISTS billing_settled_at timestamptz; + +ALTER TABLE gateway_tasks + ADD CONSTRAINT gateway_tasks_billing_status_v2_check CHECK ( + billing_status IN ( + 'not_started', 'pending', 'processing', 'settled', 'released', + 'retryable_failed', 'manual_review', 'not_required' + ) + ) NOT VALID, + ADD CONSTRAINT gateway_tasks_reservation_amount_v2_check CHECK (reservation_amount >= 0) NOT VALID; + +CREATE UNIQUE INDEX IF NOT EXISTS uniq_gateway_tasks_idempotency_hash_v2 + ON gateway_tasks(user_id, idempotency_key_hash) + WHERE idempotency_key_hash IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_gateway_tasks_execution_lease_v2 + ON gateway_tasks(status, execution_lease_expires_at) + WHERE status = 'running'; + +CREATE INDEX IF NOT EXISTS idx_gateway_tasks_billing_status_v2 + ON gateway_tasks(billing_status, billing_updated_at, created_at); + +ALTER TABLE gateway_task_attempts + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS request_fingerprint text, + ADD COLUMN IF NOT EXISTS upstream_submission_status text NOT NULL DEFAULT 'not_submitted', + ADD COLUMN IF NOT EXISTS upstream_submission_updated_at timestamptz; + +ALTER TABLE gateway_task_attempts + ADD CONSTRAINT gateway_task_attempts_submission_v2_check CHECK ( + upstream_submission_status IN ('not_submitted', 'submitting', 'response_received') + ) NOT VALID; + +ALTER TABLE settlement_outbox + ADD COLUMN IF NOT EXISTS action text NOT NULL DEFAULT 'settle', + ADD COLUMN IF NOT EXISTS amount numeric(38, 9) NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS currency text NOT NULL DEFAULT 'resource', + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS locked_by text, + ADD COLUMN IF NOT EXISTS lock_token uuid, + ADD COLUMN IF NOT EXISTS locked_at timestamptz, + ADD COLUMN IF NOT EXISTS last_error_code text, + ADD COLUMN IF NOT EXISTS last_error_message text, + ADD COLUMN IF NOT EXISTS completed_at timestamptz, + ADD COLUMN IF NOT EXISTS manual_review_reason text; + +ALTER TABLE settlement_outbox + ADD CONSTRAINT settlement_outbox_action_v2_check CHECK (action IN ('settle', 'release')) NOT VALID, + ADD CONSTRAINT settlement_outbox_status_v2_check CHECK ( + status IN ('pending', 'processing', 'retryable_failed', 'completed', 'manual_review') + ) NOT VALID, + ADD CONSTRAINT settlement_outbox_amount_v2_check CHECK (amount >= 0) NOT VALID; + +CREATE INDEX IF NOT EXISTS idx_settlement_outbox_claim_v2 + ON settlement_outbox(status, next_attempt_at, created_at) + WHERE status IN ('pending', 'retryable_failed'); + +CREATE INDEX IF NOT EXISTS idx_settlement_outbox_stale_v2 + ON settlement_outbox(status, locked_at) + WHERE status = 'processing'; + +ALTER TABLE model_pricing_rules + ADD COLUMN IF NOT EXISTS is_free boolean NOT NULL DEFAULT false; + +ALTER TABLE model_pricing_rules + ADD CONSTRAINT model_pricing_rules_explicit_free_v2_check CHECK ( + base_price > 0 OR (base_price = 0 AND is_free) + ) NOT VALID; + +ALTER TABLE gateway_wallet_accounts + ADD CONSTRAINT gateway_wallet_nonnegative_frozen_v2_check CHECK (frozen_balance >= 0) NOT VALID, + ADD CONSTRAINT gateway_wallet_balance_covers_frozen_v2_check CHECK (balance >= frozen_balance) NOT VALID; + +ALTER TABLE gateway_wallet_transactions + ADD CONSTRAINT gateway_wallet_transactions_account_restrict_v2 + FOREIGN KEY (account_id) REFERENCES gateway_wallet_accounts(id) ON DELETE RESTRICT NOT VALID; + +UPDATE gateway_tasks task +SET billing_status = 'not_required', + billing_updated_at = now() +WHERE task.run_mode <> 'production' + OR task.gateway_user_id IS NULL; + +UPDATE gateway_tasks task +SET billing_status = 'settled', + billing_currency = COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'), + billing_updated_at = now(), + billing_settled_at = COALESCE(task.finished_at, task.updated_at) +WHERE task.status = 'succeeded' + AND EXISTS ( + SELECT 1 + FROM gateway_wallet_transactions transaction + WHERE transaction.reference_type = 'gateway_task' + AND transaction.reference_id = task.id::text + AND transaction.transaction_type = 'task_billing' + ); + +UPDATE gateway_tasks task +SET billing_status = 'manual_review', + billing_currency = COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'), + billing_updated_at = now() +WHERE task.status = 'succeeded' + AND task.run_mode = 'production' + AND task.gateway_user_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM gateway_wallet_transactions transaction + WHERE transaction.reference_type = 'gateway_task' + AND transaction.reference_id = task.id::text + AND transaction.transaction_type = 'task_billing' + ); + +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, payload, status, next_attempt_at +) +SELECT task.id, + 'task.billing.release', + 'release', + transaction.amount, + account.currency, + jsonb_build_object( + 'taskId', task.id, + 'reservationIdempotencyKey', transaction.idempotency_key, + 'classification', 'historical_terminal_reservation' + ), + 'pending', + now() +FROM gateway_tasks task +JOIN gateway_wallet_transactions transaction + ON transaction.reference_type = 'gateway_task' + AND transaction.reference_id = task.id::text + AND transaction.transaction_type = 'reserve' +JOIN gateway_wallet_accounts account ON account.id = transaction.account_id +WHERE task.status IN ('failed', 'cancelled') + AND COALESCE(transaction.idempotency_key, '') <> '' + AND NOT EXISTS ( + SELECT 1 + FROM gateway_wallet_transactions release + WHERE release.account_id = transaction.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = transaction.idempotency_key || ':release' + ) +ON CONFLICT (task_id, event_type) DO NOTHING; + +UPDATE gateway_tasks task +SET billing_status = 'pending', + billing_updated_at = now() +WHERE EXISTS ( + SELECT 1 + FROM settlement_outbox outbox + WHERE outbox.task_id = task.id + AND outbox.action = 'release' + AND outbox.status IN ('pending', 'processing', 'retryable_failed') +); diff --git a/docs/billing-flow-v2.md b/docs/billing-flow-v2.md new file mode 100644 index 0000000..4944897 --- /dev/null +++ b/docs/billing-flow-v2.md @@ -0,0 +1,63 @@ +# 计费业务流程 v2 + +## 正常流程 + +```mermaid +sequenceDiagram + participant UI as 前端 + participant API as Gateway API + participant DB as PostgreSQL + participant UP as 上游模型 + participant BW as 结算 Worker + + UI->>API: 参数变化后的只读估价 + API->>DB: 读取有效计价规则 + API-->>UI: 首选预计费用 + 候选最大冻结额 + UI->>API: 正式提交（可带 Idempotency-Key） + API->>DB: 重新计价、保存快照并冻结最大金额 + API->>DB: 领取执行租约 + API->>UP: 提交已预处理请求 + UP-->>API: 成功结果与实际用量 + API->>DB: 原子写 Attempt、任务成功、最终金额和 settle Outbox + API-->>UI: succeeded + billingStatus=pending + BW->>DB: SKIP LOCKED 领取 Outbox + BW->>DB: 原子扣费、释放冻结、完成 Outbox +``` + +## 失败与取消 + +上游提交前失败、上游明确失败和用户取消均在任务终态事务内写入 `release` Outbox。Worker 使用冻结流水的幂等键释放资金。若上游请求已经进入 `submitting` 但没有可靠响应，系统不得再次提交；任务转人工复核并保留冻结。 + +## 即时估价 + +前端仅对影响计费的字段生成稳定签名，包括模型、生成模式、数量、质量、分辨率、时长、音频、参考素材和 Token 上限。签名变化后等待 350ms，取消旧请求，并用递增序号阻止乱序响应覆盖。 + +界面必须区分： + +- 计算中； +- 显式免费； +- 正常预计费用与冻结上限； +- 价格不可用； +- 估价请求失败。 + +价格不可用或估价失败时禁止生产生成；模拟模式仍允许执行并显示原因。 + +## Worker 参数 + +- 轮询间隔：1 秒。 +- 单批数量：50。 +- 处理锁过期：120 秒。 +- 最大自动尝试：20 次。 +- 退避上限：15 分钟。 + +这些参数是安全默认值，可通过受控配置调整，但不得改变幂等键、快照和账务不变量。 + +## 发布与回滚 + +`BILLING_ENGINE_MODE` 支持： + +- `observe`：并行计算并记录差异，不改变旧路径账务结果。 +- `enforce`：启用 v2 缺价拒绝、冻结与结算闭环。 +- `hold`：在调用上游前拒绝新的生产任务，已有 Outbox 继续处理。 + +发布先运行 24 小时 observe，补齐所有缺价和非法规则，再以 10% 流量启用 enforce，确认没有重复扣费、长期冻结和异常差额后扩大到 100%。回滚只切换到 hold，不回滚数据库增量，也不恢复隐式零价。 diff --git a/docs/decisions/002-billing-correctness-v2.md b/docs/decisions/002-billing-correctness-v2.md new file mode 100644 index 0000000..3de4e20 --- /dev/null +++ b/docs/decisions/002-billing-correctness-v2.md @@ -0,0 +1,86 @@ +# ADR-002：计费正确性闭环与即时估价 + +## 状态 + +Accepted + +## 日期 + +2026-07-20 + +## 背景 + +Gateway 已具备 PostgreSQL 本地钱包、任务冻结、任务扣费和钱包流水，但任务终态与钱包结算分属不同事务。进程在任务成功写入后、扣费前或释放前退出时，可能留下成功未扣费或永久冻结。现有估价只计算首选候选，缺价会退化为普通零价，且文本输出上限和精度规则与正式结算并不完全一致。 + +Comfy 的业务流程证明了三项做法有参考价值：任务状态与计费状态分离；计费操作使用独立令牌和过期接管；参数变化后立即生成只读估价。Gateway 不采用 Comfy 的三十分钟虚拟预占、结算时刷新最新价格、公开估价接口和负余额策略，因为这些做法会削弱本地钱包作为唯一账务事实源的约束。 + +## 决策 + +### 账务事实源与状态 + +Gateway PostgreSQL 本地钱包是唯一账务事实源，不与 Comfy 或其他系统双写。任务状态与计费状态独立：生成成功可以处于 `succeeded + pending`，结算失败不得篡改生成结果。 + +计费状态为： + +- `not_started`：尚未产生账务动作。 +- `pending`：结算或释放已进入 Outbox。 +- `processing`：Worker 已领取账务动作。 +- `settled`：已幂等扣费并释放冻结。 +- `released`：失败或取消任务的冻结已释放。 +- `retryable_failed`：可自动重试。 +- `manual_review`：上游结果不明或连续失败，需要人工处理。 +- `not_required`：模拟任务、无本地钱包用户或其他明确无需计费的任务。 + +### 统一计价 + +估价、提交冻结和最终账单统一使用 `effective-pricing-v2`。规则优先级为平台模型、平台、基准模型；只接受处于生效期内、状态有效、币种为 `resource` 且计算器受支持的规则。普通零价、缺价或失效规则返回 `503 pricing_unavailable`；只有 `isFree=true` 的显式免费规则允许零费用。 + +金额决策使用 9 位十进制定点数。文本费用按实际 Token 比例计算，不按千 Token 向上取整；视频以五秒为一个单位向上取整。估价对所有有效候选计算并以最大值作为冻结上限，首选候选费用继续作为 `totalAmount`。 + +前端估价是只读提示，不构成价格承诺。正式提交必须重新计价、冻结，并保存不可变计价快照。 + +### 结算闭环 + +成功 Attempt、任务成功结果、最终金额、计价快照和 `settle` Outbox 在同一个数据库事务写入；失败和取消任务原子写入 `release` Outbox。Worker 使用 `FOR UPDATE SKIP LOCKED` 批量领取，锁超时后允许其他实例接管，并在钱包行锁事务中完成幂等扣费或释放、任务计费状态和 Outbox 完成状态。 + +上游提交状态分为 `not_submitted`、`submitting` 和 `response_received`。处于 `submitting` 且结果不明的任务保留冻结并进入人工复核，不自动重试上游。 + +### 请求幂等与执行租约 + +生成接口可选接受单值 `Idempotency-Key`。数据库只保存键摘要以及规范化请求摘要；同键同请求按协议重放，同键不同请求返回 `409 idempotency_key_reused`，流式重复返回 `409 idempotency_stream_replay_unsupported`。日志和审计记录不得包含原始键或完整请求正文。 + +任务由带随机 `execution_token` 的五分钟租约领取，每三十秒续约。运行态更新和终态提交必须匹配令牌，旧 Worker 不得覆盖接管后的新结果。 + +## 账务不变量 + +1. `balance >= frozen_balance >= 0`。 +2. 同一任务和币种最多存在一笔有效扣费。 +3. 一个冻结必须最终被结算消费或释放，不能同时发生两者。 +4. 正式任务缺价时不得调用上游。 +5. 最终扣费只能使用提交时保存的计价快照，不能读取更新后的价格。 +6. 估价不得创建任务、冻结余额或写钱包流水。 +7. 历史成功未扣费任务只标记人工复核，不自动追扣。 + +## 异常矩阵 + +| 发生点 | 任务状态 | 计费状态 | 自动动作 | +| --- | --- | --- | --- | +| 计价失败 | 未创建或失败 | `not_started` | 返回 `pricing_unavailable`，不调用上游 | +| 冻结失败 | 已创建 | `not_started` | 返回余额不足，不调用上游 | +| 上游提交前失败 | 失败 | `pending` | Outbox 释放冻结 | +| 上游提交结果不明 | 运行中或失败 | `manual_review` | 保留冻结，禁止自动重试上游 | +| 上游明确失败 | 失败 | `pending` | Outbox 释放冻结 | +| 生成成功、结算待执行 | 成功 | `pending` | 返回成功结果，Worker 异步结算 | +| 结算暂时失败 | 成功 | `retryable_failed` | 指数退避重试 | +| 结算连续失败 | 成功 | `manual_review` | 管理员审计化重试 | + +## 历史数据策略 + +迁移只分类，不产生历史补扣：已有任务扣费流水的成功任务标记 `settled`；失败或取消任务的遗留冻结进入释放队列；成功但无扣费流水的生产任务进入 `manual_review`。 + +## 后果 + +- 成功结果和账务暂时失败可以独立呈现，调用方必须读取 `billingStatus`。 +- 计价规则编辑器必须要求显式免费，并在规则不可用时阻止生产生成。 +- 结算具有最终一致性，因此需要积压、延迟、重试和人工复核监控。 +- 数据库增量在回滚时保留；应用切换到 `hold` 后拒绝新的生产任务，同时继续运行结算 Worker。 -- 2.54.0 From 5114686c35ffdfdc97e697f74fa6133989c1dfbf Mon Sep 17 00:00:00 2001 From: chengcheng Date: Mon, 20 Jul 2026 23:22:45 +0800 Subject: [PATCH 02/13] =?UTF-8?q?feat(billing):=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E8=AE=A1=E4=BB=B7=E4=B8=8E=E5=80=99=E9=80=89=E5=86=BB=E7=BB=93?= =?UTF-8?q?=E4=BC=B0=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 effective-pricing-v2、9 位十进制定点计算、显式免费校验和结构化缺价错误。估价与生产预处理覆盖全部可用候选，并按最大候选费用冻结；规则优先级为平台模型、平台、基准模型。\n\n同步计价契约和 OpenAPI，补充 Token 参数别名、视频五秒向上取整及缺价回归测试。\n\n验证：go test ./...、pnpm openapi、Web 测试与构建通过。 --- apps/api/docs/swagger.json | 50 +- apps/api/docs/swagger.yaml | 38 +- apps/api/internal/httpapi/handlers.go | 13 + apps/api/internal/httpapi/openapi_models.go | 10 +- .../internal/httpapi/pricing_error_test.go | 27 + apps/api/internal/httpapi/pricing_handlers.go | 29 + .../internal/httpapi/pricing_handlers_test.go | 28 + apps/api/internal/runner/pricing.go | 36 +- apps/api/internal/runner/pricing_v2.go | 605 ++++++++++++++++++ apps/api/internal/runner/pricing_v2_test.go | 107 ++++ apps/api/internal/runner/service.go | 59 +- apps/api/internal/store/postgres.go | 3 + apps/api/internal/store/pricing_rules.go | 205 +++++- packages/contracts/src/index.ts | 10 + 14 files changed, 1192 insertions(+), 28 deletions(-) create mode 100644 apps/api/internal/httpapi/pricing_error_test.go create mode 100644 apps/api/internal/httpapi/pricing_handlers_test.go create mode 100644 apps/api/internal/runner/pricing_v2.go create mode 100644 apps/api/internal/runner/pricing_v2_test.go diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index 0c4f779..a4ea972 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -6017,6 +6017,12 @@ "schema": { "$ref": "#/definitions/httpapi.ErrorEnvelope" } + }, + "503": { + "description": "Service Unavailable", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } } } } @@ -11227,6 +11233,14 @@ "httpapi.PricingEstimateResponse": { "type": "object", "properties": { + "candidateCount": { + "type": "integer", + "example": 2 + }, + "currency": { + "type": "string", + "example": "resource" + }, "items": { "type": "array", "items": { @@ -11234,9 +11248,25 @@ "additionalProperties": true } }, + "pricingVersion": { + "type": "string", + "example": "effective-pricing-v2" + }, + "requestFingerprint": { + "type": "string", + "example": "76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f" + }, + "reservationAmount": { + "type": "number", + "example": 2.75 + }, "resolver": { "type": "string", - "example": "effective-pricing-v1" + "example": "effective-pricing-v2" + }, + "totalAmount": { + "type": "number", + "example": 1.25 } } }, @@ -13966,6 +13996,12 @@ "type": "object", "additionalProperties": {} }, + "effectiveFrom": { + "type": "string" + }, + "effectiveTo": { + "type": "string" + }, "formulaConfig": { "type": "object", "additionalProperties": {} @@ -13973,6 +14009,9 @@ "id": { "type": "string" }, + "isFree": { + "type": "boolean" + }, "metadata": { "type": "object", "additionalProperties": {} @@ -14033,10 +14072,19 @@ "type": "object", "additionalProperties": {} }, + "effectiveFrom": { + "type": "string" + }, + "effectiveTo": { + "type": "string" + }, "formulaConfig": { "type": "object", "additionalProperties": {} }, + "isFree": { + "type": "boolean" + }, "metadata": { "type": "object", "additionalProperties": {} diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index 93e7f6f..c6fc9fc 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -610,14 +610,32 @@ definitions: type: object httpapi.PricingEstimateResponse: properties: + candidateCount: + example: 2 + type: integer + currency: + example: resource + type: string items: items: additionalProperties: true type: object type: array - resolver: - example: effective-pricing-v1 + pricingVersion: + example: effective-pricing-v2 type: string + requestFingerprint: + example: 76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f + type: string + reservationAmount: + example: 2.75 + type: number + resolver: + example: effective-pricing-v2 + type: string + totalAmount: + example: 1.25 + type: number type: object httpapi.PricingRuleListResponse: properties: @@ -2485,11 +2503,17 @@ definitions: dynamicWeight: additionalProperties: {} type: object + effectiveFrom: + type: string + effectiveTo: + type: string formulaConfig: additionalProperties: {} type: object id: type: string + isFree: + type: boolean metadata: additionalProperties: {} type: object @@ -2531,9 +2555,15 @@ definitions: dynamicWeight: additionalProperties: {} type: object + effectiveFrom: + type: string + effectiveTo: + type: string formulaConfig: additionalProperties: {} type: object + isFree: + type: boolean metadata: additionalProperties: {} type: object @@ -6841,6 +6871,10 @@ paths: description: Internal Server Error schema: $ref: '#/definitions/httpapi.ErrorEnvelope' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' security: - BearerAuth: [] summary: 估算请求价格 diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 82bb8cf..2a45dad 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -892,6 +892,7 @@ func (s *Server) deleteAPIKey(w http.ResponseWriter, r *http.Request) { // @Failure 403 {object} ErrorEnvelope // @Failure 404 {object} ErrorEnvelope // @Failure 429 {object} ErrorEnvelope +// @Failure 503 {object} ErrorEnvelope // @Failure 500 {object} ErrorEnvelope // @Router /api/v1/pricing/estimate [post] func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) { @@ -916,6 +917,10 @@ func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) { } estimate, err := s.runner.Estimate(r.Context(), kind, model, body, user) if err != nil { + if runner.IsPricingUnavailable(err) { + writeErrorWithDetails(w, http.StatusServiceUnavailable, runErrorMessage(err), runErrorDetails(err), "pricing_unavailable") + return + } if errors.Is(err, store.ErrNoModelCandidate) { writeErrorWithDetails(w, statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), store.ModelCandidateErrorCode(err)) return @@ -1383,6 +1388,8 @@ func scopeForTaskKind(kind string) string { func statusFromRunError(err error) int { switch { + case runner.IsPricingUnavailable(err): + return http.StatusServiceUnavailable case clients.ErrorCode(err) == "invalid_previous_response_id" || clients.ErrorCode(err) == "response_chain_too_deep" || clients.ErrorCode(err) == "unsupported_model_protocol" || clients.ErrorCode(err) == "unsupported_response_tool" || clients.ErrorCode(err) == "unsupported_response_parameter": return http.StatusBadRequest case clients.ErrorCode(err) == "response_chain_unavailable": @@ -1411,6 +1418,9 @@ func statusFromRunError(err error) int { } func runErrorCode(err error) string { + if runner.IsPricingUnavailable(err) { + return "pricing_unavailable" + } if errors.Is(err, store.ErrNoModelCandidate) { return store.ModelCandidateErrorCode(err) } @@ -1428,6 +1438,9 @@ func runErrorMessage(err error) string { } func runErrorDetails(err error) map[string]any { + if detail := runner.PricingUnavailableDetails(err); len(detail) > 0 { + return map[string]any{"pricing": detail} + } if detail := rateLimitErrorDetail(err); len(detail) > 0 { return map[string]any{"rateLimit": detail} } diff --git a/apps/api/internal/httpapi/openapi_models.go b/apps/api/internal/httpapi/openapi_models.go index 636b91f..59bad83 100644 --- a/apps/api/internal/httpapi/openapi_models.go +++ b/apps/api/internal/httpapi/openapi_models.go @@ -188,8 +188,14 @@ type PricingEstimateRequest struct { } type PricingEstimateResponse struct { - Items []map[string]interface{} `json:"items"` - Resolver string `json:"resolver" example:"effective-pricing-v1"` + Items []map[string]interface{} `json:"items"` + Resolver string `json:"resolver" example:"effective-pricing-v2"` + TotalAmount float64 `json:"totalAmount" example:"1.25"` + ReservationAmount float64 `json:"reservationAmount" example:"2.75"` + Currency string `json:"currency" example:"resource"` + CandidateCount int `json:"candidateCount" example:"2"` + PricingVersion string `json:"pricingVersion" example:"effective-pricing-v2"` + RequestFingerprint string `json:"requestFingerprint" example:"76ef6a537de8e71bd1ca93acadc078dbdbfa9f17e45224e4f9df59f535d2886f"` } type TaskRequest struct { diff --git a/apps/api/internal/httpapi/pricing_error_test.go b/apps/api/internal/httpapi/pricing_error_test.go new file mode 100644 index 0000000..9786771 --- /dev/null +++ b/apps/api/internal/httpapi/pricing_error_test.go @@ -0,0 +1,27 @@ +package httpapi + +import ( + "net/http" + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/runner" +) + +func TestPricingUnavailableUsesStructuredServiceUnavailableError(t *testing.T) { + err := &runner.PricingUnavailableError{ + Reason: "missing, invalid, or not explicitly free", + ResourceType: "image", + RuleSetID: "rule-set-id", + } + if got := statusFromRunError(err); got != http.StatusServiceUnavailable { + t.Fatalf("statusFromRunError()=%d, want %d", got, http.StatusServiceUnavailable) + } + if got := runErrorCode(err); got != "pricing_unavailable" { + t.Fatalf("runErrorCode()=%q, want pricing_unavailable", got) + } + details := runErrorDetails(err) + pricing, _ := details["pricing"].(map[string]any) + if pricing["resourceType"] != "image" || pricing["ruleSetId"] != "rule-set-id" { + t.Fatalf("unexpected pricing details: %+v", details) + } +} diff --git a/apps/api/internal/httpapi/pricing_handlers.go b/apps/api/internal/httpapi/pricing_handlers.go index 3dad415..a0b8572 100644 --- a/apps/api/internal/httpapi/pricing_handlers.go +++ b/apps/api/internal/httpapi/pricing_handlers.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "strings" + "time" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) @@ -146,10 +147,38 @@ func validPricingRuleSetInput(input store.PricingRuleSetInput) bool { if strings.TrimSpace(input.RuleSetKey) == "" || strings.TrimSpace(input.Name) == "" || len(input.Rules) == 0 { return false } + if currency := strings.TrimSpace(input.Currency); currency != "" && currency != "resource" { + return false + } for _, rule := range input.Rules { if strings.TrimSpace(rule.ResourceType) == "" || strings.TrimSpace(rule.Unit) == "" { return false } + if rule.BasePrice < 0 || (rule.BasePrice == 0 && !rule.IsFree) { + return false + } + if currency := strings.TrimSpace(rule.Currency); currency != "" && currency != "resource" { + return false + } + switch calculator := strings.TrimSpace(rule.CalculatorType); calculator { + case "", "token_usage", "unit_weight", "duration_weight": + default: + return false + } + effectiveFrom, fromOK := pricingEffectiveTime(rule.EffectiveFrom) + effectiveTo, toOK := pricingEffectiveTime(rule.EffectiveTo) + if !fromOK || !toOK || (!effectiveFrom.IsZero() && !effectiveTo.IsZero() && !effectiveFrom.Before(effectiveTo)) { + return false + } } return true } + +func pricingEffectiveTime(value string) (time.Time, bool) { + value = strings.TrimSpace(value) + if value == "" { + return time.Time{}, true + } + parsed, err := time.Parse(time.RFC3339, value) + return parsed, err == nil +} diff --git a/apps/api/internal/httpapi/pricing_handlers_test.go b/apps/api/internal/httpapi/pricing_handlers_test.go new file mode 100644 index 0000000..4a643d3 --- /dev/null +++ b/apps/api/internal/httpapi/pricing_handlers_test.go @@ -0,0 +1,28 @@ +package httpapi + +import ( + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestPricingRuleSetRejectsImplicitZeroAndUnsupportedCalculator(t *testing.T) { + base := store.PricingRuleSetInput{ + RuleSetKey: "test", Name: "test", + Rules: []store.PricingRuleInput{{ + ResourceType: "image", Unit: "image", BasePrice: 0, + Currency: "resource", CalculatorType: "unit_weight", + }}, + } + if validPricingRuleSetInput(base) { + t.Fatal("implicit zero price must be rejected") + } + base.Rules[0].IsFree = true + if !validPricingRuleSetInput(base) { + t.Fatal("explicit free price should be accepted") + } + base.Rules[0].CalculatorType = "formula" + if validPricingRuleSetInput(base) { + t.Fatal("arbitrary formula calculator must be rejected") + } +} diff --git a/apps/api/internal/runner/pricing.go b/apps/api/internal/runner/pricing.go index abda6fb..901a866 100644 --- a/apps/api/internal/runner/pricing.go +++ b/apps/api/internal/runner/pricing.go @@ -11,10 +11,14 @@ import ( ) type EstimateResult struct { - Items []any `json:"items"` - Resolver string `json:"resolver"` - TotalAmount float64 `json:"totalAmount"` - Currency string `json:"currency"` + Items []any `json:"items"` + Resolver string `json:"resolver"` + TotalAmount float64 `json:"totalAmount"` + ReservationAmount float64 `json:"reservationAmount"` + Currency string `json:"currency"` + CandidateCount int `json:"candidateCount"` + PricingVersion string `json:"pricingVersion"` + RequestFingerprint string `json:"requestFingerprint"` } func (s *Service) Estimate(ctx context.Context, kind string, model string, body map[string]any, user *auth.User) (EstimateResult, error) { @@ -32,15 +36,21 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body if err != nil { return EstimateResult{}, err } - candidate := candidates[0] - body = preprocessRequest(kind, body, candidate) - items := s.estimatedBillings(ctx, user, kind, body, candidate) - return EstimateResult{ - Items: items, - Resolver: "effective-pricing-v1", - TotalAmount: totalBillingAmount(items), - Currency: billingCurrency(items), - }, nil + estimates := make([]candidateEstimate, 0, len(candidates)) + var pricingErr error + for _, candidate := range candidates { + candidateBody := preprocessRequest(kind, cloneMap(body), candidate) + estimate, candidateErr := s.estimateCandidateV2(ctx, user, kind, candidateBody, candidate) + if candidateErr != nil { + pricingErr = candidateErr + continue + } + estimates = append(estimates, estimate) + } + if len(estimates) == 0 && pricingErr != nil { + return EstimateResult{}, pricingErr + } + return buildEstimateResult(estimates, pricingRequestFingerprint(kind, model, body)) } func (s *Service) estimatedBillings(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) []any { diff --git a/apps/api/internal/runner/pricing_v2.go b/apps/api/internal/runner/pricing_v2.go new file mode 100644 index 0000000..95c42db --- /dev/null +++ b/apps/api/internal/runner/pricing_v2.go @@ -0,0 +1,605 @@ +package runner + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "strconv" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/clients" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +const ( + pricingVersionV2 = "effective-pricing-v2" + fixedScale = int64(1_000_000_000) +) + +var ErrPricingUnavailable = errors.New("pricing unavailable") + +type PricingUnavailableError struct { + Reason string + ResourceType string + RuleSetID string +} + +func (e *PricingUnavailableError) Error() string { + message := "pricing unavailable" + if e.ResourceType != "" { + message += " for " + e.ResourceType + } + if e.Reason != "" { + message += ": " + e.Reason + } + return message +} + +func (e *PricingUnavailableError) Unwrap() error { return ErrPricingUnavailable } + +func isPricingUnavailable(err error) bool { return errors.Is(err, ErrPricingUnavailable) } + +func IsPricingUnavailable(err error) bool { return errors.Is(err, ErrPricingUnavailable) } + +func PricingUnavailableDetails(err error) map[string]any { + var pricingErr *PricingUnavailableError + if !errors.As(err, &pricingErr) { + return nil + } + details := map[string]any{"reason": pricingErr.Reason} + if pricingErr.ResourceType != "" { + details["resourceType"] = pricingErr.ResourceType + } + if pricingErr.RuleSetID != "" { + details["ruleSetId"] = pricingErr.RuleSetID + } + return details +} + +// fixedAmount is a signed decimal with exactly nine fractional digits. +// Monetary decisions use this type; float64 is only produced at JSON-compatible boundaries. +type fixedAmount int64 + +func parseFixedAmount(value string) (fixedAmount, error) { + value = strings.TrimSpace(value) + if value == "" { + return 0, fmt.Errorf("empty decimal") + } + sign := int64(1) + if value[0] == '-' || value[0] == '+' { + if value[0] == '-' { + sign = -1 + } + value = value[1:] + } + parts := strings.Split(value, ".") + if len(parts) > 2 || value == "" { + return 0, fmt.Errorf("invalid decimal %q", value) + } + whole := parts[0] + if whole == "" { + whole = "0" + } + fraction := "" + if len(parts) == 2 { + fraction = parts[1] + } + for _, digits := range []string{whole, fraction} { + for _, char := range digits { + if char < '0' || char > '9' { + return 0, fmt.Errorf("invalid decimal %q", value) + } + } + } + roundUp := false + if len(fraction) > 9 { + roundUp = fraction[9] >= '5' + fraction = fraction[:9] + } + fraction += strings.Repeat("0", 9-len(fraction)) + combined := strings.TrimLeft(whole+fraction, "0") + if combined == "" { + combined = "0" + } + number := new(big.Int) + if _, ok := number.SetString(combined, 10); !ok { + return 0, fmt.Errorf("invalid decimal %q", value) + } + if roundUp { + number.Add(number, big.NewInt(1)) + } + if !number.IsInt64() { + return 0, fmt.Errorf("decimal %q exceeds fixed amount range", value) + } + return fixedAmount(sign * number.Int64()), nil +} + +func fixedAmountFromAny(value any) (fixedAmount, error) { + switch typed := value.(type) { + case fixedAmount: + return typed, nil + case string: + return parseFixedAmount(typed) + case json.Number: + return parseFixedAmount(typed.String()) + case int: + return fixedAmount(int64(typed) * fixedScale), nil + case int64: + return fixedAmount(typed * fixedScale), nil + case int32: + return fixedAmount(int64(typed) * fixedScale), nil + case float64: + if math.IsNaN(typed) || math.IsInf(typed, 0) { + return 0, fmt.Errorf("non-finite decimal") + } + return parseFixedAmount(strconv.FormatFloat(typed, 'f', 9, 64)) + case float32: + return fixedAmountFromAny(float64(typed)) + default: + return 0, fmt.Errorf("unsupported decimal type %T", value) + } +} + +func (a fixedAmount) String() string { + value := int64(a) + sign := "" + if value < 0 { + sign = "-" + value = -value + } + return fmt.Sprintf("%s%d.%09d", sign, value/fixedScale, value%fixedScale) +} + +func (a fixedAmount) Float64() float64 { return float64(a) / float64(fixedScale) } +func (a fixedAmount) IsZero() bool { return a == 0 } + +func (a fixedAmount) Add(other fixedAmount) fixedAmount { return a + other } + +func (a fixedAmount) MulInt(multiplier int) fixedAmount { + return fixedAmount(int64(a) * int64(multiplier)) +} + +func (a fixedAmount) Mul(other fixedAmount) fixedAmount { + product := new(big.Int).Mul(big.NewInt(int64(a)), big.NewInt(int64(other))) + return fixedAmount(roundBigIntRatio(product, big.NewInt(fixedScale))) +} + +func (a fixedAmount) MulRatio(numerator int, denominator int) fixedAmount { + if denominator == 0 { + return 0 + } + product := new(big.Int).Mul(big.NewInt(int64(a)), big.NewInt(int64(numerator))) + return fixedAmount(roundBigIntRatio(product, big.NewInt(int64(denominator)))) +} + +func roundBigIntRatio(numerator *big.Int, denominator *big.Int) int64 { + negative := numerator.Sign() < 0 + absolute := new(big.Int).Abs(new(big.Int).Set(numerator)) + quotient, remainder := new(big.Int), new(big.Int) + quotient.QuoRem(absolute, denominator, remainder) + if new(big.Int).Mul(remainder, big.NewInt(2)).Cmp(denominator) >= 0 { + quotient.Add(quotient, big.NewInt(1)) + } + if negative { + quotient.Neg(quotient) + } + return quotient.Int64() +} + +type resolvedPricing struct { + Config map[string]any + Currency string + RuleSetID string + RuleSetKey string + Source string + FreeResource map[string]bool + Snapshot map[string]any +} + +func (pricing resolvedPricing) requiredPrice(resource string, keys ...string) (fixedAmount, error) { + if price, found, err := pricing.price(resource, keys...); err != nil { + return 0, &PricingUnavailableError{Reason: err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID} + } else if found && price > 0 { + return price, nil + } else if found && price == 0 && pricing.FreeResource[resource] { + return 0, nil + } + return 0, &PricingUnavailableError{Reason: "missing, invalid, or not explicitly free", ResourceType: resource, RuleSetID: pricing.RuleSetID} +} + +func (pricing resolvedPricing) price(resource string, keys ...string) (fixedAmount, bool, error) { + for _, key := range keys { + if value, ok := pricing.Config[key]; ok { + amount, err := fixedAmountFromAny(value) + return amount, true, err + } + } + resourceConfig, _ := pricing.Config[resource].(map[string]any) + if len(resourceConfig) == 0 && resource == "image_edit" { + resourceConfig, _ = pricing.Config["image"].(map[string]any) + } + for _, key := range keys { + if value, ok := resourceConfig[key]; ok { + amount, err := fixedAmountFromAny(value) + return amount, true, err + } + } + if formula, ok := resourceConfig["formulaConfig"].(map[string]any); ok { + for _, key := range keys { + if value, exists := formula[key]; exists { + amount, err := fixedAmountFromAny(value) + return amount, true, err + } + } + } + if value, ok := resourceConfig["basePrice"]; ok { + amount, err := fixedAmountFromAny(value) + return amount, true, err + } + if resource == "text" { + return resolvedPricing{Config: pricing.Config}.price("text_total", keys...) + } + return 0, false, nil +} + +type candidateEstimate struct { + Items []any + Amount fixedAmount + Currency string + Snapshot map[string]any + Pricing resolvedPricing +} + +func buildEstimateResult(estimates []candidateEstimate, fingerprint string) (EstimateResult, error) { + if len(estimates) == 0 { + return EstimateResult{}, &PricingUnavailableError{Reason: "no candidate has effective pricing"} + } + preferred := estimates[0] + reservation := preferred.Amount + currency := preferred.Currency + if currency == "" { + currency = "resource" + } + for _, estimate := range estimates[1:] { + candidateCurrency := estimate.Currency + if candidateCurrency == "" { + candidateCurrency = "resource" + } + if candidateCurrency != currency { + return EstimateResult{}, &PricingUnavailableError{Reason: "candidate currencies do not match"} + } + if estimate.Amount > reservation { + reservation = estimate.Amount + } + } + return EstimateResult{ + Items: preferred.Items, + Resolver: pricingVersionV2, + TotalAmount: preferred.Amount.Float64(), + ReservationAmount: reservation.Float64(), + Currency: currency, + CandidateCount: len(estimates), + PricingVersion: pricingVersionV2, + RequestFingerprint: fingerprint, + }, nil +} + +func estimatedOutputTokens(body map[string]any, candidate store.RuntimeModelCandidate) int { + for _, key := range []string{"max_completion_tokens", "max_output_tokens", "max_tokens"} { + if value, ok := positiveInteger(body[key]); ok { + return value + } + } + if limit, _, _, ok := candidateMaxOutputTokens(candidate, candidate.ModelType); ok { + return limit + } + return 4096 +} + +func pricingRequestFingerprint(kind string, model string, body map[string]any) string { + payload, _ := json.Marshal(map[string]any{"kind": kind, "model": model, "request": body}) + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func (s *Service) resolveEffectivePricing(ctx context.Context, candidate store.RuntimeModelCandidate) (resolvedPricing, error) { + ruleSetID := firstNonEmptyString( + candidate.ModelPricingRuleSetID, + candidate.PlatformPricingRuleSetID, + candidate.BasePricingRuleSetID, + ) + if ruleSetID != "" { + if s.store == nil { + return resolvedPricing{}, &PricingUnavailableError{Reason: "pricing store is unavailable", RuleSetID: ruleSetID} + } + config, err := s.store.PricingRuleSetBillingConfigV2(ctx, ruleSetID) + if err != nil { + return resolvedPricing{}, &PricingUnavailableError{Reason: err.Error(), RuleSetID: ruleSetID} + } + if len(candidate.BillingConfigOverride) > 0 { + config.Config = mergeMap(config.Config, candidate.BillingConfigOverride) + } + return resolvedPricing{ + Config: config.Config, Currency: config.Currency, RuleSetID: config.RuleSetID, + RuleSetKey: config.RuleSetKey, Source: pricingRuleSource(candidate, ruleSetID), + FreeResource: config.FreeResource, Snapshot: config.Snapshot, + }, nil + } + + config := candidate.BaseBillingConfig + source := "base_model_config" + if len(candidate.BillingConfig) > 0 { + config = candidate.BillingConfig + source = "platform_model_config" + } + if len(candidate.BillingConfigOverride) > 0 { + config = mergeMap(config, candidate.BillingConfigOverride) + source += "_override" + } + if len(config) == 0 { + return resolvedPricing{}, &PricingUnavailableError{Reason: "candidate has no pricing configuration"} + } + freeResource := explicitFreeResources(config) + snapshot := map[string]any{ + "pricingVersion": pricingVersionV2, + "source": source, + "currency": "resource", + "config": config, + } + return resolvedPricing{ + Config: config, Currency: "resource", Source: source, + FreeResource: freeResource, Snapshot: snapshot, + }, nil +} + +func pricingRuleSource(candidate store.RuntimeModelCandidate, ruleSetID string) string { + switch ruleSetID { + case candidate.ModelPricingRuleSetID: + return "platform_model_rule_set" + case candidate.PlatformPricingRuleSetID: + return "platform_rule_set" + default: + return "base_model_rule_set" + } +} + +func explicitFreeResources(config map[string]any) map[string]bool { + resources := map[string]bool{} + if isFree, _ := config["isFree"].(bool); isFree { + for _, resource := range []string{"text_input", "text_cached_input", "text_output", "text_total", "image", "image_edit", "video", "music", "audio"} { + resources[resource] = true + } + } + for _, resource := range []string{"text", "text_total", "image", "image_edit", "video", "music", "audio"} { + resourceConfig, _ := config[resource].(map[string]any) + if isFree, _ := resourceConfig["isFree"].(bool); isFree { + resources[resource] = true + } + } + return resources +} + +func (s *Service) estimateCandidateV2(ctx context.Context, user *auth.User, kind string, body map[string]any, candidate store.RuntimeModelCandidate) (candidateEstimate, error) { + usage := clients.Usage{InputTokens: estimateRequestTokens(body)} + if isTextGenerationKind(kind) { + usage.OutputTokens = estimatedOutputTokens(body, candidate) + } + usage.TotalTokens = usage.InputTokens + usage.OutputTokens + response := clients.Response{Usage: usage} + items, total, pricing, err := s.billingsV2(ctx, user, kind, body, candidate, response, true) + if err != nil { + return candidateEstimate{}, err + } + if isTextBillingKind(kind) { + if _, err := pricing.requiredTextPrice("text_cached_input", "textCachedInputPer1k", "cachedInputTokenPrice", "cacheInputTokenPrice", "textInputCacheHitPer1k", "inputCacheHitTokenPrice"); err != nil { + return candidateEstimate{}, err + } + } + return candidateEstimate{ + Items: items, Amount: total, Currency: pricing.Currency, Snapshot: pricing.Snapshot, Pricing: pricing, + }, nil +} + +func (s *Service) billingsV2( + ctx context.Context, + user *auth.User, + kind string, + body map[string]any, + candidate store.RuntimeModelCandidate, + response clients.Response, + simulated bool, +) ([]any, fixedAmount, resolvedPricing, error) { + pricing, err := s.resolveEffectivePricing(ctx, candidate) + if err != nil { + return nil, 0, resolvedPricing{}, err + } + return s.billingsWithResolvedPricingV2(ctx, user, kind, body, candidate, response, simulated, pricing) +} + +func (s *Service) billingsWithResolvedPricingV2( + ctx context.Context, + user *auth.User, + kind string, + body map[string]any, + candidate store.RuntimeModelCandidate, + response clients.Response, + simulated bool, + pricing resolvedPricing, +) ([]any, fixedAmount, resolvedPricing, error) { + discount, err := fixedAmountFromAny(effectiveDiscount(ctx, s.store, user, candidate)) + if err != nil || discount <= 0 { + return nil, 0, resolvedPricing{}, &PricingUnavailableError{Reason: "invalid discount factor", RuleSetID: pricing.RuleSetID} + } + buildLine := func(resourceType string, unit string, quantity any, amount fixedAmount, details map[string]any) map[string]any { + line := billingLineWithDetails(candidate, resourceType, unit, quantity, amount.Float64(), discount.Float64(), simulated, details) + line["pricingVersion"] = pricingVersionV2 + line["pricingSource"] = pricing.Source + if pricing.RuleSetID != "" { + line["pricingRuleSetId"] = pricing.RuleSetID + } + line["isFree"] = amount.IsZero() + return line + } + + if isTextBillingKind(kind) { + inputTokens := response.Usage.InputTokens + outputTokens := response.Usage.OutputTokens + cachedInputTokens := response.Usage.CachedInputTokens + if isTextInputOnlyKind(kind) && inputTokens == 0 && response.Usage.TotalTokens > 0 { + inputTokens = response.Usage.TotalTokens + } + if inputTokens == 0 && outputTokens == 0 { + inputTokens = estimateRequestTokens(body) + if isTextGenerationKind(kind) { + outputTokens = 1 + } + } + if cachedInputTokens > inputTokens && inputTokens > 0 { + cachedInputTokens = inputTokens + } + uncachedInputTokens := inputTokens - cachedInputTokens + if uncachedInputTokens < 0 { + uncachedInputTokens = 0 + } + inputPrice, err := pricing.requiredTextPrice("text_input", "textInputPer1k", "inputTokenPrice", "basePrice") + if err != nil { + return nil, 0, resolvedPricing{}, err + } + items := make([]any, 0, 3) + total := fixedAmount(0) + if uncachedInputTokens > 0 || cachedInputTokens == 0 { + amount := inputPrice.MulRatio(uncachedInputTokens, 1000).Mul(discount) + total = total.Add(amount) + items = append(items, buildLine("text_input", "1k_tokens", uncachedInputTokens, amount, map[string]any{ + "inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens, + "cachedInputTokens": cachedInputTokens, "pricePer1k": inputPrice.Float64(), + })) + } + if cachedInputTokens > 0 { + cachedPrice, priceErr := pricing.requiredTextPrice("text_cached_input", "textCachedInputPer1k", "cachedInputTokenPrice", "cacheInputTokenPrice", "textInputCacheHitPer1k", "inputCacheHitTokenPrice") + if priceErr != nil { + return nil, 0, resolvedPricing{}, priceErr + } + amount := cachedPrice.MulRatio(cachedInputTokens, 1000).Mul(discount) + total = total.Add(amount) + items = append(items, buildLine("text_cached_input", "1k_tokens", cachedInputTokens, amount, map[string]any{ + "inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens, + "cachedInputTokens": cachedInputTokens, "pricePer1k": cachedPrice.Float64(), + })) + } + if isTextGenerationKind(kind) { + outputPrice, priceErr := pricing.requiredTextPrice("text_output", "textOutputPer1k", "outputTokenPrice", "basePrice") + if priceErr != nil { + return nil, 0, resolvedPricing{}, priceErr + } + amount := outputPrice.MulRatio(outputTokens, 1000).Mul(discount) + total = total.Add(amount) + items = append(items, buildLine("text_output", "1k_tokens", outputTokens, amount, map[string]any{"pricePer1k": outputPrice.Float64()})) + } + return items, total, pricing, nil + } + + count := requestOutputCount(body) + resource := "image" + unit := "image" + baseKey := "imageBase" + if kind == "images.edits" { + resource = "image_edit" + baseKey = "editBase" + } + if kind == "videos.generations" { + resource = "video" + unit = "5s_video" + baseKey = "videoBase" + duration, durationSource := billingDurationSeconds(body, response) + audioEnabled, audioSource := billingAudioEnabled(body, response) + durationUnits := int(math.Max(1, math.Ceil(duration/5))) + price, priceErr := pricing.requiredPrice(resource, baseKey, "basePrice") + if priceErr != nil { + return nil, 0, resolvedPricing{}, priceErr + } + amount := price.MulInt(count).MulInt(durationUnits). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))))). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "audioWeights", boolWeightKey(audioEnabled)))). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "referenceVideoWeights", boolWeightKey(requestHasReferenceVideo(body))))). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "voiceSpecifiedWeights", boolWeightKey(requestHasVoiceID(body, audioEnabled))))). + Mul(discount) + item := buildLine(resource, unit, count*durationUnits, amount, map[string]any{ + "count": count, "audio": audioEnabled, "audioSource": audioSource, + "durationSeconds": duration, "durationSource": durationSource, + "durationUnit": "5s", "durationUnitCount": durationUnits, + }) + return []any{item}, amount, pricing, nil + } + if kind == "song.generations" || kind == "music.generations" { + resource = "music" + unit = "song" + baseKey = "musicBase" + } + if kind == "speech.generations" || kind == "voice.clone" { + resource = "audio" + unit = "character" + baseKey = "audioBase" + count = len([]rune(stringFromMap(body, "text"))) + if count <= 0 { + count = 1 + } + } + price, err := pricing.requiredPrice(resource, baseKey, "basePrice") + if err != nil { + return nil, 0, resolvedPricing{}, err + } + amount := price.MulInt(count) + if resource == "image" || resource == "image_edit" { + amount = amount. + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "qualityWeights", stringFromMap(body, "quality")))). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "sizeWeights", stringFromMap(body, "size")))). + Mul(fixedWeight(resourceWeight(pricing.Config, resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))))) + } + amount = amount.Mul(discount) + return []any{buildLine(resource, unit, count, amount, nil)}, amount, pricing, nil +} + +func (pricing resolvedPricing) requiredTextPrice(resource string, keys ...string) (fixedAmount, error) { + price, found, err := pricing.price("text", keys...) + if err != nil { + return 0, &PricingUnavailableError{Reason: err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID} + } + if found && price > 0 { + return price, nil + } + if found && price == 0 && (pricing.FreeResource[resource] || pricing.FreeResource["text_total"] || pricing.FreeResource["text"]) { + return 0, nil + } + return 0, &PricingUnavailableError{Reason: "missing, invalid, or not explicitly free", ResourceType: resource, RuleSetID: pricing.RuleSetID} +} + +func fixedWeight(value float64) fixedAmount { + weight, err := fixedAmountFromAny(value) + if err != nil || weight <= 0 { + return fixedAmount(fixedScale) + } + return weight +} + +func maximumCandidateEstimate(estimates []candidateEstimate) (candidateEstimate, error) { + if len(estimates) == 0 { + return candidateEstimate{}, &PricingUnavailableError{Reason: "no candidate has effective pricing"} + } + maximum := estimates[0] + for _, estimate := range estimates[1:] { + if estimate.Currency != maximum.Currency { + return candidateEstimate{}, &PricingUnavailableError{Reason: "candidate currencies do not match"} + } + if estimate.Amount > maximum.Amount { + maximum = estimate + } + } + return maximum, nil +} diff --git a/apps/api/internal/runner/pricing_v2_test.go b/apps/api/internal/runner/pricing_v2_test.go new file mode 100644 index 0000000..ea16827 --- /dev/null +++ b/apps/api/internal/runner/pricing_v2_test.go @@ -0,0 +1,107 @@ +package runner + +import ( + "testing" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestFixedAmountPreservesNineDecimalPlaces(t *testing.T) { + amount, err := parseFixedAmount("0.123456789") + if err != nil { + t.Fatalf("parse fixed amount: %v", err) + } + if got, want := amount.String(), "0.123456789"; got != want { + t.Fatalf("amount.String()=%q, want %q", got, want) + } + + price := mustFixedAmount(t, "0.000000001") + if got, want := price.MulInt(3).String(), "0.000000003"; got != want { + t.Fatalf("nano amount multiplication=%q, want %q", got, want) + } +} + +func TestEstimatedOutputTokensUsesAliasesAndCapabilityFallback(t *testing.T) { + candidate := store.RuntimeModelCandidate{ + ModelType: "text_generate", + Capabilities: map[string]any{ + "text_generate": map[string]any{"max_output_tokens": 8192}, + }, + } + tests := []struct { + name string + body map[string]any + want int + }{ + {name: "max completion has highest priority", body: map[string]any{"max_completion_tokens": 700, "max_output_tokens": 600, "max_tokens": 500}, want: 700}, + {name: "max output precedes legacy max tokens", body: map[string]any{"max_output_tokens": 600, "max_tokens": 500}, want: 600}, + {name: "legacy max tokens", body: map[string]any{"max_tokens": 500}, want: 500}, + {name: "model capability fallback", body: map[string]any{}, want: 8192}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := estimatedOutputTokens(test.body, candidate); got != test.want { + t.Fatalf("estimatedOutputTokens()=%d, want %d", got, test.want) + } + }) + } + + if got := estimatedOutputTokens(nil, store.RuntimeModelCandidate{}); got != 4096 { + t.Fatalf("missing capability fallback=%d, want 4096", got) + } +} + +func TestBuildEstimateResultUsesPreferredTotalAndMaximumReservation(t *testing.T) { + result, err := buildEstimateResult([]candidateEstimate{ + {Items: []any{map[string]any{"amount": 1.25, "currency": "resource"}}, Amount: mustFixedAmount(t, "1.25")}, + {Items: []any{map[string]any{"amount": 2.75, "currency": "resource"}}, Amount: mustFixedAmount(t, "2.75")}, + }, "fingerprint") + if err != nil { + t.Fatalf("build estimate result: %v", err) + } + if result.TotalAmount != 1.25 { + t.Fatalf("preferred total=%v, want 1.25", result.TotalAmount) + } + if result.ReservationAmount != 2.75 { + t.Fatalf("reservation=%v, want 2.75", result.ReservationAmount) + } + if result.CandidateCount != 2 || result.PricingVersion != pricingVersionV2 || result.RequestFingerprint != "fingerprint" { + t.Fatalf("unexpected estimate metadata: %+v", result) + } +} + +func TestPricingAvailabilityRequiresExplicitFree(t *testing.T) { + paid := resolvedPricing{Config: map[string]any{"imageBase": 1.5}, Currency: "resource"} + if _, err := paid.requiredPrice("image", "imageBase", "basePrice"); err != nil { + t.Fatalf("positive price should be available: %v", err) + } + + missing := resolvedPricing{Config: map[string]any{}, Currency: "resource"} + if _, err := missing.requiredPrice("image", "imageBase", "basePrice"); !isPricingUnavailable(err) { + t.Fatalf("missing price should be unavailable, got %v", err) + } + + ordinaryZero := resolvedPricing{Config: map[string]any{"imageBase": 0}, Currency: "resource"} + if _, err := ordinaryZero.requiredPrice("image", "imageBase", "basePrice"); !isPricingUnavailable(err) { + t.Fatalf("ordinary zero should be unavailable, got %v", err) + } + + explicitFree := resolvedPricing{ + Config: map[string]any{"imageBase": 0}, + Currency: "resource", + FreeResource: map[string]bool{"image": true}, + } + price, err := explicitFree.requiredPrice("image", "imageBase", "basePrice") + if err != nil || !price.IsZero() { + t.Fatalf("explicit free should resolve to zero: price=%s err=%v", price.String(), err) + } +} + +func mustFixedAmount(t *testing.T, value string) fixedAmount { + t.Helper() + amount, err := parseFixedAmount(value) + if err != nil { + t.Fatalf("parse %q: %v", value, err) + } + return amount +} diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 6461363..3ab59e1 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -258,6 +258,44 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut return Result{Task: failed, Output: failed.Result}, err } } + pricingByCandidate := map[string]resolvedPricing{} + reservationBillings := []any(nil) + if task.RunMode == "production" { + pricedCandidates := make([]store.RuntimeModelCandidate, 0, len(candidates)) + estimates := make([]candidateEstimate, 0, len(candidates)) + var pricingErr error + for _, candidate := range candidates { + pricingBody := preprocessRequest(task.Kind, cloneMap(body), candidate) + estimate, estimateErr := s.estimateCandidateV2(ctx, user, task.Kind, pricingBody, candidate) + if estimateErr != nil { + pricingErr = estimateErr + continue + } + pricedCandidates = append(pricedCandidates, candidate) + estimates = append(estimates, estimate) + pricingByCandidate[pricingCandidateKey(candidate)] = estimate.Pricing + } + if len(pricedCandidates) == 0 { + if pricingErr == nil { + pricingErr = &PricingUnavailableError{Reason: "no candidate has effective pricing"} + } + failed, finishErr := s.failTask(ctx, task.ID, "pricing_unavailable", pricingErr.Error(), false, pricingErr) + if finishErr != nil { + return Result{}, finishErr + } + return Result{Task: failed, Output: failed.Result}, pricingErr + } + maximumEstimate, estimateErr := maximumCandidateEstimate(estimates) + if estimateErr != nil { + failed, finishErr := s.failTask(ctx, task.ID, "pricing_unavailable", estimateErr.Error(), false, estimateErr) + if finishErr != nil { + return Result{}, finishErr + } + return Result{Task: failed, Output: failed.Result}, estimateErr + } + candidates = pricedCandidates + reservationBillings = maximumEstimate.Items + } firstCandidateBody := body normalizedModelType := modelType attemptNo := task.AttemptCount @@ -299,9 +337,8 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut if err := s.store.MarkTaskRunning(ctx, task.ID, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil { return Result{}, err } - estimatedBillings := s.estimatedBillings(ctx, user, task.Kind, firstCandidateBody, candidates[0]) var reserveErr error - walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, estimatedBillings) + walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings) if reserveErr != nil { if errors.Is(reserveErr, store.ErrInsufficientWalletBalance) { attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{ @@ -372,7 +409,19 @@ candidatesLoop: response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record) if err == nil { attemptNo = nextAttemptNo - billings := s.billings(ctx, user, task.Kind, candidateBody, candidate, response, isSimulation(task, candidate)) + var billings []any + if task.RunMode == "production" { + pricing := pricingByCandidate[pricingCandidateKey(candidate)] + var billingErr error + billings, _, _, billingErr = s.billingsWithResolvedPricingV2(ctx, user, task.Kind, candidateBody, candidate, response, false, pricing) + if billingErr != nil { + // The upstream result may already exist. Preserve the reservation for manual review. + walletReservationFinalized = true + return Result{}, billingErr + } + } else { + billings = s.billings(ctx, user, task.Kind, candidateBody, candidate, response, true) + } record := buildSuccessRecord(task, user, candidateBody, candidate, response, billings, isSimulation(task, candidate)) record.Metrics = mergeMetrics(record.Metrics, candidateCapabilityFilterMetrics(candidateFilterSummary)) record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log)) @@ -580,6 +629,10 @@ candidatesLoop: return Result{Task: failed, Output: failed.Result}, lastErr } +func pricingCandidateKey(candidate store.RuntimeModelCandidate) string { + return firstNonEmptyString(candidate.PlatformModelID, candidate.PlatformID+":"+candidate.ModelName) +} + func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) { simulated := isSimulation(task, candidate) baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing)) diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index 2d32011..c364834 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -295,6 +295,7 @@ type PricingRule struct { ResourceType string `json:"resourceType"` Unit string `json:"unit"` BasePrice float64 `json:"basePrice"` + IsFree bool `json:"isFree"` Currency string `json:"currency"` BaseWeight map[string]any `json:"baseWeight,omitempty"` DynamicWeight map[string]any `json:"dynamicWeight,omitempty"` @@ -304,6 +305,8 @@ type PricingRule struct { Priority int `json:"priority"` Status string `json:"status"` Metadata map[string]any `json:"metadata,omitempty"` + EffectiveFrom string `json:"effectiveFrom,omitempty"` + EffectiveTo string `json:"effectiveTo,omitempty"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` } diff --git a/apps/api/internal/store/pricing_rules.go b/apps/api/internal/store/pricing_rules.go index 8e4efbc..a3136cc 100644 --- a/apps/api/internal/store/pricing_rules.go +++ b/apps/api/internal/store/pricing_rules.go @@ -3,8 +3,10 @@ package store import ( "context" "encoding/json" + "fmt" "strconv" "strings" + "time" "github.com/jackc/pgx/v5" ) @@ -15,9 +17,10 @@ status, metadata, created_at, updated_at` const pricingRuleColumns = ` id::text, COALESCE(rule_set_id::text, ''), rule_key, display_name, scope_type, -COALESCE(scope_id::text, ''), resource_type, unit, base_price::float8, currency, +COALESCE(scope_id::text, ''), resource_type, unit, base_price::float8, is_free, currency, base_weight, dynamic_weight, calculator_type, dimension_schema, formula_config, -priority, status, metadata, created_at, updated_at` +priority, status, metadata, COALESCE(effective_from::text, ''), COALESCE(effective_to::text, ''), +created_at, updated_at` type PricingRuleInput struct { RuleKey string `json:"ruleKey"` @@ -25,6 +28,7 @@ type PricingRuleInput struct { ResourceType string `json:"resourceType"` Unit string `json:"unit"` BasePrice float64 `json:"basePrice"` + IsFree bool `json:"isFree"` Currency string `json:"currency"` BaseWeight map[string]any `json:"baseWeight"` DynamicWeight map[string]any `json:"dynamicWeight"` @@ -34,6 +38,17 @@ type PricingRuleInput struct { Priority int `json:"priority"` Status string `json:"status"` Metadata map[string]any `json:"metadata"` + EffectiveFrom string `json:"effectiveFrom"` + EffectiveTo string `json:"effectiveTo"` +} + +type EffectivePricingConfig struct { + RuleSetID string + RuleSetKey string + Currency string + Config map[string]any + FreeResource map[string]bool + Snapshot map[string]any } type PricingRuleSetInput struct { @@ -247,6 +262,173 @@ ORDER BY priority ASC, resource_type ASC`, id) return config, nil } +func (s *Store) PricingRuleSetBillingConfigV2(ctx context.Context, id string) (EffectivePricingConfig, error) { + id = strings.TrimSpace(id) + if id == "" { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule set id is required") + } + var ruleSetKey string + var currency string + var status string + if err := s.pool.QueryRow(ctx, ` +SELECT rule_set_key, currency, status +FROM model_pricing_rule_sets +WHERE id = $1::uuid`, id).Scan(&ruleSetKey, ¤cy, &status); err != nil { + return EffectivePricingConfig{}, err + } + if status != "active" { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s is not active", id) + } + if currency != "resource" { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s uses unsupported currency %s", id, currency) + } + + rows, err := s.pool.Query(ctx, ` +SELECT rule_key, resource_type, unit, base_price::text, currency, base_weight, + dynamic_weight, calculator_type, dimension_schema, formula_config, + priority, is_free, COALESCE(effective_from::text, ''), COALESCE(effective_to::text, '') +FROM model_pricing_rules +WHERE rule_set_id = $1::uuid + AND status = 'active' + AND (effective_from IS NULL OR effective_from <= now()) + AND (effective_to IS NULL OR effective_to > now()) +ORDER BY priority ASC, resource_type ASC, rule_key ASC`, id) + if err != nil { + return EffectivePricingConfig{}, err + } + defer rows.Close() + + config := map[string]any{} + freeResource := map[string]bool{} + rules := make([]any, 0) + seen := map[string]bool{} + for rows.Next() { + var ruleKey string + var resourceType string + var unit string + var basePrice string + var ruleCurrency string + var baseWeightBytes []byte + var dynamicWeightBytes []byte + var calculatorType string + var dimensionSchemaBytes []byte + var formulaConfigBytes []byte + var priority int + var isFree bool + var effectiveFrom string + var effectiveTo string + if err := rows.Scan( + &ruleKey, &resourceType, &unit, &basePrice, &ruleCurrency, &baseWeightBytes, + &dynamicWeightBytes, &calculatorType, &dimensionSchemaBytes, &formulaConfigBytes, + &priority, &isFree, &effectiveFrom, &effectiveTo, + ); err != nil { + return EffectivePricingConfig{}, err + } + if seen[resourceType] { + continue + } + if ruleCurrency != currency { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s currency does not match its rule set", ruleKey) + } + switch calculatorType { + case "token_usage", "unit_weight", "duration_weight": + default: + return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s uses unsupported calculator %s", ruleKey, calculatorType) + } + if strings.HasPrefix(strings.TrimSpace(basePrice), "-") { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s has a negative base price", ruleKey) + } + dynamicWeight := decodeObject(dynamicWeightBytes) + formulaConfig := decodeObject(formulaConfigBytes) + addPricingRuleToConfig(config, resourceType, basePrice, dynamicWeight, formulaConfig) + freeResource[resourceType] = isFree + seen[resourceType] = true + rules = append(rules, map[string]any{ + "ruleKey": ruleKey, "resourceType": resourceType, "unit": unit, + "basePrice": basePrice, "currency": ruleCurrency, "baseWeight": decodeObject(baseWeightBytes), + "dynamicWeight": dynamicWeight, "calculatorType": calculatorType, + "dimensionSchema": decodeObject(dimensionSchemaBytes), "formulaConfig": formulaConfig, + "priority": priority, "isFree": isFree, + "effectiveFrom": effectiveFrom, "effectiveTo": effectiveTo, + }) + } + if err := rows.Err(); err != nil { + return EffectivePricingConfig{}, err + } + if len(rules) == 0 { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule set %s has no effective rules", id) + } + return EffectivePricingConfig{ + RuleSetID: id, RuleSetKey: ruleSetKey, Currency: currency, Config: config, + FreeResource: freeResource, + Snapshot: map[string]any{ + "pricingVersion": "effective-pricing-v2", + "ruleSetId": id, + "ruleSetKey": ruleSetKey, + "currency": currency, + "resolvedAt": time.Now().UTC().Format(time.RFC3339Nano), + "rules": rules, + }, + }, nil +} + +func addPricingRuleToConfig(config map[string]any, resourceType string, basePrice string, dynamicWeight map[string]any, formulaConfig map[string]any) { + resourceConfig := map[string]any{"basePrice": basePrice} + if len(dynamicWeight) > 0 { + resourceConfig["dynamicWeight"] = dynamicWeight + } + if len(formulaConfig) > 0 { + resourceConfig["formulaConfig"] = formulaConfig + } + switch resourceType { + case "text_input": + config["textInputPer1k"] = basePrice + case "text_cached_input": + config["textCachedInputPer1k"] = basePrice + case "text_output": + config["textOutputPer1k"] = basePrice + case "text_total": + inputPrice := any(basePrice) + if value, ok := pricingRuleValueFromKeys(formulaConfig, "inputTokenPrice", "input_token_price", "textInputPer1k", "text_input"); ok { + inputPrice = value + } + config["textInputPer1k"] = inputPrice + if value, ok := pricingRuleValueFromKeys(formulaConfig, "cachedInputTokenPrice", "cached_input_token_price", "textCachedInputPer1k", "text_cached_input", "inputCacheHitTokenPrice", "input_cache_hit_token_price"); ok { + config["textCachedInputPer1k"] = value + } + if value, ok := pricingRuleValueFromKeys(formulaConfig, "outputTokenPrice", "output_token_price", "textOutputPer1k", "text_output"); ok { + config["textOutputPer1k"] = value + } + config["text_total"] = resourceConfig + case "image": + config["imageBase"] = basePrice + config["image"] = resourceConfig + case "image_edit": + config["editBase"] = basePrice + config["image_edit"] = resourceConfig + case "video": + config["videoBase"] = basePrice + config["video"] = resourceConfig + case "music": + config["musicBase"] = basePrice + config["music"] = resourceConfig + case "audio": + config["audioBase"] = basePrice + config["audio"] = resourceConfig + default: + config[resourceType] = resourceConfig + } +} + +func pricingRuleValueFromKeys(config map[string]any, keys ...string) (any, bool) { + for _, key := range keys { + if value, ok := config[key]; ok { + return value, true + } + } + return nil, false +} + func pricingRuleNumberFromKeys(config map[string]any, keys ...string) (float64, bool) { if len(config) == 0 { return 0, false @@ -305,13 +487,16 @@ func insertPricingRules(ctx context.Context, tx pgx.Tx, ruleSetID string, defaul if _, err := tx.Exec(ctx, ` INSERT INTO model_pricing_rules ( rule_set_id, rule_key, display_name, scope_type, scope_id, resource_type, - unit, base_price, currency, base_weight, dynamic_weight, calculator_type, - dimension_schema, formula_config, priority, status, metadata + unit, base_price, is_free, currency, base_weight, dynamic_weight, calculator_type, + dimension_schema, formula_config, priority, status, metadata, effective_from, effective_to ) -VALUES ($1::uuid, $2, $3, 'rule_set', $1::uuid, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)`, +VALUES ( + $1::uuid, $2, $3, 'rule_set', $1::uuid, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, NULLIF($17, '')::timestamptz, NULLIF($18, '')::timestamptz +)`, ruleSetID, rule.RuleKey, rule.DisplayName, rule.ResourceType, rule.Unit, - rule.BasePrice, rule.Currency, baseWeight, dynamicWeight, rule.CalculatorType, - dimensionSchema, formulaConfig, rule.Priority, rule.Status, metadata, + rule.BasePrice, rule.IsFree, rule.Currency, baseWeight, dynamicWeight, rule.CalculatorType, + dimensionSchema, formulaConfig, rule.Priority, rule.Status, metadata, rule.EffectiveFrom, rule.EffectiveTo, ); err != nil { return err } @@ -358,6 +543,7 @@ func scanPricingRule(scanner pricingScanner) (PricingRule, error) { &item.ResourceType, &item.Unit, &item.BasePrice, + &item.IsFree, &item.Currency, &baseWeight, &dynamicWeight, @@ -367,6 +553,8 @@ func scanPricingRule(scanner pricingScanner) (PricingRule, error) { &item.Priority, &item.Status, &metadata, + &item.EffectiveFrom, + &item.EffectiveTo, &item.CreatedAt, &item.UpdatedAt, ); err != nil { @@ -444,6 +632,7 @@ func pricingInputsToRules(ruleSetID string, defaultCurrency string, rules []Pric ResourceType: input.ResourceType, Unit: input.Unit, BasePrice: input.BasePrice, + IsFree: input.IsFree, Currency: input.Currency, BaseWeight: emptyObjectIfNil(input.BaseWeight), DynamicWeight: emptyObjectIfNil(input.DynamicWeight), @@ -453,6 +642,8 @@ func pricingInputsToRules(ruleSetID string, defaultCurrency string, rules []Pric Priority: input.Priority, Status: input.Status, Metadata: emptyObjectIfNil(input.Metadata), + EffectiveFrom: input.EffectiveFrom, + EffectiveTo: input.EffectiveTo, }) } return items diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 5f0472f..ea9504d 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -184,6 +184,7 @@ export interface PricingRule { | string; unit: '1k_tokens' | 'image' | '5s' | 'second' | 'character_1k' | 'item' | string; basePrice: number; + isFree: boolean; currency: 'resource' | 'credit' | 'cny' | 'usd' | string; baseWeight?: Record; dynamicWeight?: Record; @@ -193,6 +194,8 @@ export interface PricingRule { priority: number; status: 'active' | 'deprecated' | 'hidden' | string; metadata?: Record; + effectiveFrom?: string; + effectiveTo?: string; createdAt: string; updatedAt: string; } @@ -203,6 +206,7 @@ export interface PricingRuleInput { resourceType: string; unit: string; basePrice: number; + isFree?: boolean; currency?: string; baseWeight?: Record; dynamicWeight?: Record; @@ -212,6 +216,8 @@ export interface PricingRuleInput { priority?: number; status?: 'active' | 'deprecated' | 'hidden' | string; metadata?: Record; + effectiveFrom?: string; + effectiveTo?: string; } export interface PricingRuleSet { @@ -548,7 +554,11 @@ export interface GatewayPricingEstimate { items: GatewayPricingEstimateItem[]; resolver: string; totalAmount?: number; + reservationAmount?: number; currency?: string; + candidateCount: number; + pricingVersion: string; + requestFingerprint: string; } export interface GatewayWalletAccount { -- 2.54.0 From 7cea21f76539ba8ed2f3b800c1333a9f8cfc983d Mon Sep 17 00:00:00 2001 From: chengcheng Date: Mon, 20 Jul 2026 23:22:50 +0800 Subject: [PATCH 03/13] =?UTF-8?q?feat(web):=20=E5=A2=9E=E5=8A=A0=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E5=8D=B3=E6=97=B6=E4=BC=B0=E4=BB=B7=E4=B8=8E=E6=98=BE?= =?UTF-8?q?=E5=BC=8F=E5=85=8D=E8=B4=B9=E7=BC=96=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 估价请求统一使用 350ms 防抖、AbortController 和请求序号，区分免费、计算中、价格不可用与失败状态，并展示候选冻结上限。价格未就绪时阻止生产提交。\n\n计价规则编辑器新增显式免费开关和 9 位价格步进。\n\n验证：Web 88 项测试和生产构建通过。 --- apps/web/src/api.ts | 5 +- .../src/pages/PlaygroundPage.pricing.test.ts | 49 ++++++++ apps/web/src/pages/PlaygroundPage.tsx | 116 +++++++++++++++--- .../pages/admin/PricingRuleVisualEditor.tsx | 41 ++++++- .../web/src/pages/admin/PricingRulesPanel.tsx | 3 + 5 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 apps/web/src/pages/PlaygroundPage.pricing.test.ts diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index 0946295..f367ae3 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -896,10 +896,12 @@ export async function uploadFileToStorage( export async function estimatePricing( token: string, input: Record, + signal?: AbortSignal, ): Promise { return request('/api/v1/pricing/estimate', { body: input, method: 'POST', + signal, token, }); } @@ -1180,7 +1182,7 @@ export async function deleteFileStorageChannel(token: string, channelId: string) async function request( path: string, - options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record } = {}, + options: { token?: string; auth?: boolean; method?: string; body?: unknown; headers?: Record; signal?: AbortSignal } = {}, ): Promise { const headers: Record = { ...(options.headers ?? {}) }; if (options.auth !== false && options.token && options.token !== OIDC_BROWSER_SESSION_CREDENTIAL) { @@ -1194,6 +1196,7 @@ async function request( headers, body: options.body === undefined ? undefined : JSON.stringify(options.body), credentials: 'include', + signal: options.signal, }); if (!response.ok) { const body = await response.text(); diff --git a/apps/web/src/pages/PlaygroundPage.pricing.test.ts b/apps/web/src/pages/PlaygroundPage.pricing.test.ts new file mode 100644 index 0000000..abacfb2 --- /dev/null +++ b/apps/web/src/pages/PlaygroundPage.pricing.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import type { GatewayPricingEstimate } from '@easyai-ai-gateway/contracts'; +import { + MEDIA_ESTIMATE_DEBOUNCE_MS, + billingEstimateSignature, + mediaEstimateFromResponse, +} from './PlaygroundPage'; + +describe('playground pricing estimate', () => { + it('uses the specified 350ms debounce interval', () => { + expect(MEDIA_ESTIMATE_DEBOUNCE_MS).toBe(350); + }); + + it('only changes its signature for billing-relevant parameters', () => { + const first = billingEstimateSignature({ + kind: 'images.generations', model: 'image-model', prompt: 'first', + count: 1, quality: 'high', resolution: '2K', + }); + const promptOnly = billingEstimateSignature({ + kind: 'images.generations', model: 'image-model', prompt: 'second', + count: 1, quality: 'high', resolution: '2K', + }); + const changedCount = billingEstimateSignature({ + kind: 'images.generations', model: 'image-model', prompt: 'second', + count: 2, quality: 'high', resolution: '2K', + }); + expect(promptOnly).toBe(first); + expect(changedCount).not.toBe(first); + }); + + it('distinguishes explicit free and exposes the reservation ceiling', () => { + const response: GatewayPricingEstimate = { + candidateCount: 2, + currency: 'resource', + items: [{ amount: 0, currency: 'resource', isFree: true }], + pricingVersion: 'effective-pricing-v2', + requestFingerprint: 'fingerprint', + reservationAmount: 0, + resolver: 'effective-pricing-v2', + totalAmount: 0, + }; + expect(mediaEstimateFromResponse(response)).toMatchObject({ + amount: 0, + candidateCount: 2, + reservationAmount: 0, + status: 'free', + }); + }); +}); diff --git a/apps/web/src/pages/PlaygroundPage.tsx b/apps/web/src/pages/PlaygroundPage.tsx index a9b8af1..048bc6c 100644 --- a/apps/web/src/pages/PlaygroundPage.tsx +++ b/apps/web/src/pages/PlaygroundPage.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; import type { GatewayApiKey, GatewayPricingEstimate, GatewayTask, PlatformModel } from '@easyai-ai-gateway/contracts'; import { ArrowUp, ChevronDown, MessageSquarePlus, Settings2, Sparkles } from 'lucide-react'; import { Badge, Button, FormDialog, Select, Textarea } from '../components/ui'; -import { createImageEditTask, createImageGenerationTask, createVideoGenerationTask, estimatePricing, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api'; +import { GatewayApiError, createImageEditTask, createImageGenerationTask, createVideoGenerationTask, estimatePricing, pollTaskUntilSettled, resolveApiAssetUrl, taskIsPending } from '../api'; import type { PlaygroundMode } from '../types'; import { PlaygroundPromptMentionInput, @@ -56,13 +56,17 @@ import { const MEDIA_RUNS_STORAGE_KEY = 'easyai:playground:media-runs:v1'; const MEDIA_RUNS_STORAGE_LIMIT = 50; +export const MEDIA_ESTIMATE_DEBOUNCE_MS = 350; type MediaEstimateState = { amount?: number; + reservationAmount?: number; + candidateCount?: number; currency?: string; error?: string; + pricingVersion?: string; resolver?: string; - status: 'idle' | 'loading' | 'ready' | 'error'; + status: 'idle' | 'loading' | 'ready' | 'free' | 'unavailable' | 'error'; }; const publicWorks = [ @@ -107,6 +111,7 @@ export function PlaygroundPage(props: { const [settingsOpen, setSettingsOpen] = useState(false); const isMountedRef = useRef(false); const pendingMediaModelRef = useRef(''); + const mediaEstimateSequenceRef = useRef(0); const resumedTaskIdsRef = useRef(new Set()); const activeMode = useMemo(() => modeOptions.find((item) => item.value === props.mode) ?? modeOptions[0], [props.mode]); const mediaUploadAcceptValue = sharedMediaUploadAcceptForMode(props.mode, videoMode); @@ -136,6 +141,10 @@ export function PlaygroundPage(props: { supportsQualityControl: mediaCapabilities?.supportsQualityControl, }); }, [mediaCapabilities, mediaSettings, mediaUploads, prompt, props.mode, selectedModel, videoMode]); + const mediaEstimateSignature = useMemo( + () => billingEstimateSignature(mediaEstimatePayload), + [mediaEstimatePayload], + ); useEffect(() => { setSelectedModel((current) => { @@ -176,30 +185,34 @@ export function PlaygroundPage(props: { useEffect(() => { const credential = activeApiKeySecret || props.token; if (props.mode === 'chat' || !credential || !mediaEstimatePayload) { + mediaEstimateSequenceRef.current += 1; setMediaEstimate({ status: 'idle' }); return; } - let cancelled = false; + const sequence = mediaEstimateSequenceRef.current + 1; + mediaEstimateSequenceRef.current = sequence; + const controller = new AbortController(); setMediaEstimate((current) => ({ ...current, error: '', status: 'loading' })); const timer = window.setTimeout(() => { - estimatePricing(credential, mediaEstimatePayload) + estimatePricing(credential, mediaEstimatePayload, controller.signal) .then((estimate) => { - if (cancelled) return; + if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return; setMediaEstimate(mediaEstimateFromResponse(estimate)); }) .catch((err) => { - if (cancelled) return; + if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return; + const unavailable = err instanceof GatewayApiError && err.details.code === 'pricing_unavailable'; setMediaEstimate({ - error: err instanceof Error ? err.message : '预计扣费计算失败', - status: 'error', + error: err instanceof Error ? err.message : unavailable ? '价格不可用' : '预计扣费计算失败', + status: unavailable ? 'unavailable' : 'error', }); }); - }, 260); + }, MEDIA_ESTIMATE_DEBOUNCE_MS); return () => { - cancelled = true; window.clearTimeout(timer); + controller.abort(); }; - }, [activeApiKeySecret, mediaEstimatePayload, props.mode, props.token]); + }, [activeApiKeySecret, mediaEstimateSignature, props.mode, props.token]); useEffect(() => { if (props.mode === 'image') { @@ -313,6 +326,12 @@ export function PlaygroundPage(props: { setMediaMessage(runMode === 'video' ? '请输入视频提示词。' : '请输入图片提示词。'); return; } + if (!overrides && !mediaEstimateAllowsProduction(mediaEstimate)) { + setMediaMessage(mediaEstimate.status === 'unavailable' + ? '当前参数没有可用价格，已阻止生产生成。' + : '请等待当前参数的预计费用计算完成后再生成。'); + return; + } const localId = newLocalId(); const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads); @@ -456,6 +475,7 @@ export function PlaygroundPage(props: { imageHasReference={effectiveImageHasReference} mediaSettings={mediaSettings} mediaEstimate={mediaEstimate} + submitDisabled={!mediaEstimateAllowsProduction(mediaEstimate)} mediaCapabilities={mediaCapabilities} uploadAccept={mediaUploadAcceptValue} uploadMessage={mediaUploadMessage} @@ -684,6 +704,7 @@ function Composer(props: { imageHasReference?: boolean; mediaCapabilities?: MediaModelCapabilities; mediaEstimate?: MediaEstimateState; + submitDisabled?: boolean; mediaSettings?: MediaGenerationSettings; mode: PlaygroundMode; modelOptions: ModelOption[]; @@ -791,7 +812,7 @@ function Composer(props: { {mediaEstimateText(props.mediaEstimate)} )} - @@ -828,12 +849,17 @@ function buildMediaEstimatePayload( }; } -function mediaEstimateFromResponse(response: GatewayPricingEstimate): MediaEstimateState { +export function mediaEstimateFromResponse(response: GatewayPricingEstimate): MediaEstimateState { + const amount = numericFromUnknown(response.totalAmount) ?? estimateItemsTotal(response.items); + const explicitFree = amount === 0 && response.items.length > 0 && response.items.every((item) => item.isFree === true); return { - amount: numericFromUnknown(response.totalAmount) ?? estimateItemsTotal(response.items), + amount, + reservationAmount: numericFromUnknown(response.reservationAmount), + candidateCount: response.candidateCount, currency: response.currency || estimateItemsCurrency(response.items), + pricingVersion: response.pricingVersion, resolver: response.resolver, - status: 'ready', + status: explicitFree ? 'free' : 'ready', }; } @@ -847,21 +873,79 @@ function estimateItemsCurrency(items: GatewayPricingEstimate['items']) { } function mediaEstimateText(estimate: MediaEstimateState) { + if (estimate.status === 'loading') return '计算中'; + if (estimate.status === 'unavailable') return '价格不可用'; + if (estimate.status === 'error') return '估价失败'; + if (estimate.status === 'free') return '免费'; if (estimate.amount === undefined) return '--'; - return formatEstimateAmount(estimate.amount); + const amount = formatEstimateAmount(estimate.amount); + const reservation = estimate.reservationAmount; + if (reservation !== undefined && reservation > estimate.amount) { + return `预计 ${amount} · 冻结上限 ${formatEstimateAmount(reservation)}`; + } + return `预计 ${amount}`; } function mediaEstimateHint(estimate: MediaEstimateState) { + if (estimate.status === 'unavailable' && estimate.error) return `价格不可用：${estimate.error}`; if (estimate.status === 'error' && estimate.error) return `预计扣费计算失败：${estimate.error}`; + if (estimate.status === 'free') return '当前计价规则已显式标记为免费'; return '扣费为预计，实际扣费以账单为准'; } function mediaEstimateAriaLabel(estimate: MediaEstimateState) { if (estimate.status === 'error') return estimate.error ? `预计扣费计算失败：${estimate.error}` : '预计扣费计算失败'; + if (estimate.status === 'unavailable') return estimate.error ? `价格不可用：${estimate.error}` : '价格不可用'; + if (estimate.status === 'free') return '当前请求免费'; if (estimate.amount === undefined) return '预计扣费估算中'; return `预计扣费 ${formatEstimateAmount(estimate.amount)} ${estimate.currency || 'resource'}`; } +function mediaEstimateAllowsProduction(estimate: MediaEstimateState) { + return estimate.status === 'ready' || estimate.status === 'free'; +} + +export function billingEstimateSignature(payload: Record | null) { + if (!payload) return ''; + const content = Array.isArray(payload.content) + ? payload.content.map((item) => { + const record = item && typeof item === 'object' ? item as Record : {}; + return { + duration: record.duration, + role: record.role, + type: record.type, + hasAudio: Boolean(record.audio_url), + hasImage: Boolean(record.image_url), + hasVideo: Boolean(record.video_url), + }; + }) + : undefined; + const signature = { + kind: payload.kind, + model: payload.model, + n: payload.n ?? payload.count ?? payload.batch_size ?? payload.batchSize, + quality: payload.quality, + size: payload.size, + resolution: payload.resolution, + duration: payload.duration, + audio: payload.audio, + maxCompletionTokens: payload.max_completion_tokens, + maxOutputTokens: payload.max_output_tokens, + maxTokens: payload.max_tokens, + content, + imageCount: pricingReferenceCount(payload, ['image', 'images', 'image_url', 'image_urls', 'referenceImage', 'reference_image']), + }; + return JSON.stringify(signature); +} + +function pricingReferenceCount(payload: Record, keys: string[]) { + return keys.reduce((count, key) => { + const value = payload[key]; + if (Array.isArray(value)) return count + value.length; + return count + (value ? 1 : 0); + }, 0); +} + function formatEstimateAmount(value: number) { if (!Number.isFinite(value)) return '--'; const digits = Math.abs(value) > 0 && Math.abs(value) < 0.01 ? 6 : 2; diff --git a/apps/web/src/pages/admin/PricingRuleVisualEditor.tsx b/apps/web/src/pages/admin/PricingRuleVisualEditor.tsx index 336d47b..2e45e1a 100644 --- a/apps/web/src/pages/admin/PricingRuleVisualEditor.tsx +++ b/apps/web/src/pages/admin/PricingRuleVisualEditor.tsx @@ -221,7 +221,17 @@ function ModeRuleEditor(props: { - patch({ basePrice: Number(event.target.value) })} /> + patch({ basePrice: Number(event.target.value) })} /> + + + @@ -311,11 +321,32 @@ function TextRuleEditor(props: { + + + @@ -329,9 +360,10 @@ function TextRuleEditor(props: { @@ -345,9 +377,10 @@ function TextRuleEditor(props: { diff --git a/apps/web/src/pages/admin/PricingRulesPanel.tsx b/apps/web/src/pages/admin/PricingRulesPanel.tsx index 7bf44f1..d696ac3 100644 --- a/apps/web/src/pages/admin/PricingRulesPanel.tsx +++ b/apps/web/src/pages/admin/PricingRulesPanel.tsx @@ -225,6 +225,7 @@ function ruleToInput(rule: PricingRule): PricingRuleInput { resourceType: rule.resourceType, unit: rule.unit, basePrice: rule.basePrice, + isFree: rule.isFree, currency: rule.currency, baseWeight: rule.baseWeight ?? {}, dynamicWeight: rule.dynamicWeight ?? {}, @@ -234,6 +235,8 @@ function ruleToInput(rule: PricingRule): PricingRuleInput { priority: rule.priority, status: rule.status, metadata: rule.metadata ?? {}, + effectiveFrom: rule.effectiveFrom, + effectiveTo: rule.effectiveTo, }; } -- 2.54.0 From 5b2b94b1bdf8d2d622022be950f713b4cb5a4a38 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 00:25:53 +0800 Subject: [PATCH 04/13] =?UTF-8?q?feat(billing):=20=E5=AE=8C=E6=88=90?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E7=BB=93=E7=AE=97=E4=B8=8E=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E9=97=AD=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 统一任务成功、Attempt 与结算 Outbox 的事务边界，增加多实例安全结算、人工复核、请求幂等与执行租约。钱包决策使用九位精确金额，并通过审计保护约束保留流水事实；同时补充管理接口、指标与 PostgreSQL 集成测试。 --- .env.example | 6 + .../internal/clients/openai_request_params.go | 5 + .../clients/openai_request_params_test.go | 17 + apps/api/internal/clients/types.go | 1 + apps/api/internal/config/config.go | 7 + .../httpapi/billing_admin_handlers.go | 22 +- .../httpapi/billing_settlement_handlers.go | 145 ++++ .../billing_settlement_handlers_test.go | 21 + .../httpapi/core_flow_integration_test.go | 53 +- apps/api/internal/httpapi/gemini_compat.go | 37 +- apps/api/internal/httpapi/handlers.go | 44 +- .../httpapi/oidc_jit_integration_test.go | 20 +- apps/api/internal/httpapi/openapi_models.go | 12 + apps/api/internal/httpapi/server.go | 9 +- apps/api/internal/httpapi/task_idempotency.go | 85 +++ .../internal/httpapi/task_idempotency_test.go | 31 + apps/api/internal/runner/billing_mode_test.go | 27 + .../runner/billing_settlement_worker.go | 96 +++ .../runner/billing_settlement_worker_test.go | 40 ++ apps/api/internal/runner/execution_lease.go | 28 + apps/api/internal/runner/pricing.go | 7 +- apps/api/internal/runner/queue_worker.go | 22 +- apps/api/internal/runner/service.go | 355 +++++++--- .../internal/runner/wallet_execute_test.go | 14 + apps/api/internal/securityevents/metrics.go | 88 ++- .../api/internal/store/billing_settlements.go | 637 ++++++++++++++++++ .../store/billing_v2_integration_test.go | 265 ++++++++ apps/api/internal/store/postgres.go | 303 ++++++--- apps/api/internal/store/runtime_types.go | 62 +- apps/api/internal/store/tasks_runtime.go | 384 +++++++++-- apps/api/internal/store/wallet.go | 394 +++++++++-- .../internal/store/wallet_reservation_test.go | 4 + .../0069_billing_correctness_v2.sql | 61 +- 33 files changed, 2884 insertions(+), 418 deletions(-) create mode 100644 apps/api/internal/httpapi/billing_settlement_handlers.go create mode 100644 apps/api/internal/httpapi/billing_settlement_handlers_test.go create mode 100644 apps/api/internal/httpapi/task_idempotency.go create mode 100644 apps/api/internal/httpapi/task_idempotency_test.go create mode 100644 apps/api/internal/runner/billing_mode_test.go create mode 100644 apps/api/internal/runner/billing_settlement_worker.go create mode 100644 apps/api/internal/runner/billing_settlement_worker_test.go create mode 100644 apps/api/internal/runner/execution_lease.go create mode 100644 apps/api/internal/store/billing_settlements.go create mode 100644 apps/api/internal/store/billing_v2_integration_test.go diff --git a/.env.example b/.env.example index 6439feb..6965c88 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,12 @@ CONFIG_JWT_SECRET=this is a very secret secret # - hybrid: both sources are accepted and separated by gateway_users.source. IDENTITY_MODE=hybrid +# Billing engine rollout mode: +# - observe: keep legacy billing decisions and compare effective-pricing-v2 in logs. +# - enforce: require v2 pricing, reserve the candidate maximum, then settle asynchronously. +# - hold: reject new production generation before any upstream request; existing settlements continue. +BILLING_ENGINE_MODE=observe + # Unified identity business settings are managed in System Settings > Unified # Identity. Deployment only supplies the SecretStore and infrastructure timing. AI_GATEWAY_PUBLIC_BASE_URL=http://localhost:8088 diff --git a/apps/api/internal/clients/openai_request_params.go b/apps/api/internal/clients/openai_request_params.go index 388a8cb..062d6e3 100644 --- a/apps/api/internal/clients/openai_request_params.go +++ b/apps/api/internal/clients/openai_request_params.go @@ -34,6 +34,11 @@ var gatewayOpenAIRequestExtensions = stringSet( "requestId", "request_id", "signal", "userMessage", "user_message", "platformId", "platform_id", "options", "enable_thinking", "thinking_budget_tokens", "enable_web_search", "modelType", "model_type", "capability", "capabilityType", "mode", "simulation", "testMode", + "cacheAffinityKey", "cache_affinity_key", "simulationDurationMs", "simulationDurationSeconds", + "simulationMinDurationMs", "simulationMaxDurationMs", "simulationMinDurationSeconds", + "simulationMaxDurationSeconds", "simulationDurationMinMs", "simulationDurationMaxMs", + "simulationDurationMinSeconds", "simulationDurationMaxSeconds", "simulationFailure", + "simulationProfile", "simulationUsage", ) var gatewayResponsesRequestExtensions = stringSet("messages", "presence_penalty", "frequency_penalty") diff --git a/apps/api/internal/clients/openai_request_params_test.go b/apps/api/internal/clients/openai_request_params_test.go index df4de80..931a0ff 100644 --- a/apps/api/internal/clients/openai_request_params_test.go +++ b/apps/api/internal/clients/openai_request_params_test.go @@ -60,6 +60,23 @@ func TestValidateOpenAIRequestParametersRejectsUnknownTopLevelField(t *testing.T } } +func TestValidateOpenAIRequestParametersAcceptsInternalSimulationAndAffinityFields(t *testing.T) { + body := map[string]any{ + "model": "demo", "messages": []any{}, "simulation": true, + "simulationDurationMs": 5, "simulationUsage": map[string]any{"inputTokens": 10}, + "cacheAffinityKey": "conversation-1", + } + if err := ValidateOpenAIRequestParameters("chat.completions", body); err != nil { + t.Fatalf("expected documented gateway extensions to remain accepted, got %v", err) + } + filtered := FilterOpenAIChatRequestBody(body) + for _, key := range []string{"simulation", "simulationDurationMs", "simulationUsage", "cacheAffinityKey"} { + if _, ok := filtered[key]; ok { + t.Fatalf("gateway-only field %q leaked upstream", key) + } + } +} + func TestResponsesFallbackMapsEquivalentCurrentParameters(t *testing.T) { body, err := ResponsesRequestToChat(map[string]any{ "input": "hello", "store": false, "metadata": map[string]any{"trace": "1"}, diff --git a/apps/api/internal/clients/types.go b/apps/api/internal/clients/types.go index 5462f05..6f1db88 100644 --- a/apps/api/internal/clients/types.go +++ b/apps/api/internal/clients/types.go @@ -36,6 +36,7 @@ type ResponseTurn struct { } type Response struct { + AttemptID string Result map[string]any RequestID string Usage Usage diff --git a/apps/api/internal/config/config.go b/apps/api/internal/config/config.go index f373963..728aa62 100644 --- a/apps/api/internal/config/config.go +++ b/apps/api/internal/config/config.go @@ -47,6 +47,7 @@ type Config struct { GlobalHTTPProxy string GlobalHTTPProxySource string LogLevel slog.Level + BillingEngineMode string } func Load() Config { @@ -90,10 +91,16 @@ func Load() Config { GlobalHTTPProxy: globalProxy.HTTPProxy, GlobalHTTPProxySource: globalProxy.Source, LogLevel: logLevel(env("LOG_LEVEL", "info")), + BillingEngineMode: strings.ToLower(env("BILLING_ENGINE_MODE", "observe")), } } func (c Config) Validate() error { + switch strings.ToLower(strings.TrimSpace(c.BillingEngineMode)) { + case "", "observe", "enforce", "hold": + default: + return errors.New("BILLING_ENGINE_MODE must be observe, enforce, or hold") + } switch strings.ToLower(strings.TrimSpace(c.IdentitySecretStore)) { case "": case "file": diff --git a/apps/api/internal/httpapi/billing_admin_handlers.go b/apps/api/internal/httpapi/billing_admin_handlers.go index 5c0ef38..a6c995d 100644 --- a/apps/api/internal/httpapi/billing_admin_handlers.go +++ b/apps/api/internal/httpapi/billing_admin_handlers.go @@ -13,7 +13,7 @@ import ( type walletBalanceRequest struct { Currency string `json:"currency" example:"USD"` - Balance float64 `json:"balance" example:"100"` + Balance json.Number `json:"balance" swaggertype:"number" example:"100"` Reason string `json:"reason" example:"manual recharge"` IdempotencyKey string `json:"idempotencyKey" example:"wallet-set-20260514-001"` Metadata map[string]any `json:"metadata"` @@ -21,7 +21,7 @@ type walletBalanceRequest struct { type walletRechargeRequest struct { Currency string `json:"currency" example:"resource"` - Amount float64 `json:"amount" example:"100"` + Amount json.Number `json:"amount" swaggertype:"number" example:"100"` Reason string `json:"reason" example:"manual recharge"` IdempotencyKey string `json:"idempotencyKey" example:"wallet-recharge-20260514-001"` Metadata map[string]any `json:"metadata"` @@ -50,10 +50,6 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid json body") return } - if input.Balance < 0 { - writeError(w, http.StatusBadRequest, "wallet balance cannot be negative") - return - } gatewayUserID := strings.TrimSpace(r.PathValue("userID")) reason := strings.TrimSpace(input.Reason) if reason == "" { @@ -67,7 +63,7 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) { next, err := s.store.SetUserWalletBalanceTx(r.Context(), tx, store.WalletBalanceAdjustmentInput{ GatewayUserID: gatewayUserID, Currency: input.Currency, - Balance: input.Balance, + BalanceText: input.Balance.String(), Reason: reason, IdempotencyKey: input.IdempotencyKey, Metadata: input.Metadata, @@ -89,6 +85,10 @@ func (s *Server) setUserWalletBalance(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusNotFound, "user not found") case errors.Is(err, store.ErrWalletBalanceUnchanged): writeError(w, http.StatusBadRequest, "wallet balance is unchanged") + case errors.Is(err, store.ErrInvalidWalletAmount): + writeError(w, http.StatusBadRequest, "wallet balance must be a non-negative decimal with at most nine fractional digits", "invalid_wallet_amount") + case errors.Is(err, store.ErrBalanceBelowFrozen): + writeError(w, http.StatusConflict, "wallet balance cannot be below frozen balance", "balance_below_frozen") default: s.logger.Error("set user wallet balance failed", "error", err) writeError(w, http.StatusInternalServerError, "set user wallet balance failed") @@ -126,10 +126,6 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques writeError(w, http.StatusBadRequest, "invalid json body") return } - if input.Amount <= 0 { - writeError(w, http.StatusBadRequest, "wallet recharge amount must be positive") - return - } gatewayUserID := strings.TrimSpace(r.PathValue("userID")) reason := strings.TrimSpace(input.Reason) if reason == "" { @@ -143,7 +139,7 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques next, err := s.store.RechargeUserWalletBalanceTx(r.Context(), tx, store.WalletRechargeInput{ GatewayUserID: gatewayUserID, Currency: input.Currency, - Amount: input.Amount, + AmountText: input.Amount.String(), Reason: reason, IdempotencyKey: input.IdempotencyKey, Metadata: input.Metadata, @@ -163,6 +159,8 @@ func (s *Server) rechargeUserWalletBalance(w http.ResponseWriter, r *http.Reques switch { case store.IsNotFound(err): writeError(w, http.StatusNotFound, "user not found") + case errors.Is(err, store.ErrInvalidWalletAmount): + writeError(w, http.StatusBadRequest, "wallet recharge amount must be a positive decimal with at most nine fractional digits", "invalid_wallet_amount") default: s.logger.Error("recharge user wallet balance failed", "error", err) writeError(w, http.StatusInternalServerError, "recharge user wallet balance failed") diff --git a/apps/api/internal/httpapi/billing_settlement_handlers.go b/apps/api/internal/httpapi/billing_settlement_handlers.go new file mode 100644 index 0000000..2085069 --- /dev/null +++ b/apps/api/internal/httpapi/billing_settlement_handlers.go @@ -0,0 +1,145 @@ +package httpapi + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" +) + +// listBillingSettlements godoc +// @Summary 查询计费结算队列 +// @Description 管理端分页查询待结算、重试失败和人工复核记录。 +// @Tags billing +// @Produce json +// @Security BearerAuth +// @Param status query string false "结算状态" +// @Param action query string false "动作：settle 或 release" +// @Param page query int false "页码" default(1) +// @Param pageSize query int false "每页数量" default(50) +// @Success 200 {object} BillingSettlementListResponse +// @Failure 400 {object} ErrorEnvelope +// @Failure 401 {object} ErrorEnvelope +// @Failure 403 {object} ErrorEnvelope +// @Failure 500 {object} ErrorEnvelope +// @Router /api/admin/runtime/billing-settlements [get] +func (s *Server) listBillingSettlements(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + page, err := positiveQueryInt(query.Get("page"), 1) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid page", "invalid_request") + return + } + pageSize, err := positiveQueryInt(query.Get("pageSize"), 50) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid pageSize", "invalid_request") + return + } + result, err := s.store.ListBillingSettlements(r.Context(), store.BillingSettlementListFilter{ + Status: query.Get("status"), Action: query.Get("action"), Page: page, PageSize: pageSize, + }) + if err != nil { + s.logger.Error("list billing settlements failed", "error_category", "billing_settlement_list_failed") + writeError(w, http.StatusInternalServerError, "list billing settlements failed", "billing_settlement_list_failed") + return + } + writeJSON(w, http.StatusOK, result) +} + +// retryBillingSettlement godoc +// @Summary 重试计费结算 +// @Description Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。 +// @Tags billing +// @Produce json +// @Security BearerAuth +// @Param settlementId path string true "结算记录 ID" +// @Param Idempotency-Key header string true "幂等键" +// @Success 200 {object} BillingSettlementRetryResponse +// @Failure 400 {object} ErrorEnvelope +// @Failure 401 {object} ErrorEnvelope +// @Failure 403 {object} ErrorEnvelope +// @Failure 404 {object} ErrorEnvelope +// @Failure 409 {object} ErrorEnvelope +// @Failure 500 {object} ErrorEnvelope +// @Router /api/admin/runtime/billing-settlements/{settlementId}/retry [post] +func (s *Server) retryBillingSettlement(w http.ResponseWriter, r *http.Request) { + key, ok := singleIdempotencyKey(r) + if !ok { + writeError(w, http.StatusBadRequest, "a single Idempotency-Key is required", "idempotency_key_required") + return + } + keyHash := sha256.Sum256([]byte(key)) + settlementID := strings.TrimSpace(r.PathValue("settlementId")) + if _, err := uuid.Parse(settlementID); err != nil { + writeError(w, http.StatusBadRequest, "invalid billing settlement ID", "invalid_request") + return + } + actor, _ := auth.UserFromContext(r.Context()) + var item store.BillingSettlement + var audit store.AuditLog + var replayed bool + err := s.store.InTx(r.Context(), func(tx store.Tx) error { + var err error + item, replayed, err = s.store.RetryBillingSettlementTx(r.Context(), tx, settlementID, hex.EncodeToString(keyHash[:])) + if err != nil || replayed { + return err + } + audit, err = s.store.RecordAuditLogTx(r.Context(), tx, billingSettlementRetryAuditInput(r, actor, item)) + return err + }) + if err != nil { + switch { + case store.IsNotFound(err): + writeError(w, http.StatusNotFound, "billing settlement not found", "billing_settlement_not_found") + case errors.Is(err, store.ErrBillingSettlementNotRetryable): + writeError(w, http.StatusConflict, "billing settlement is not retryable", "billing_settlement_not_retryable") + default: + s.logger.Error("retry billing settlement failed", "settlementID", settlementID, "error_category", "billing_settlement_retry_failed") + writeError(w, http.StatusInternalServerError, "retry billing settlement failed", "billing_settlement_retry_failed") + } + return + } + if replayed { + w.Header().Set("Idempotent-Replayed", "true") + } + response := map[string]any{"settlement": item} + if !replayed { + response["auditLog"] = audit + } + writeJSON(w, http.StatusOK, response) +} + +func singleIdempotencyKey(r *http.Request) (string, bool) { + values := r.Header.Values("Idempotency-Key") + if len(values) != 1 { + return "", false + } + value := strings.TrimSpace(values[0]) + if value == "" || len(value) > 255 || strings.Contains(value, ",") { + return "", false + } + return value, true +} + +func billingSettlementRetryAuditInput(r *http.Request, actor *auth.User, item store.BillingSettlement) store.AuditLogInput { + input := store.AuditLogInput{ + Category: "billing", Action: "billing.settlement.retry", + TargetType: "billing_settlement", TargetID: item.ID, + RequestIP: requestIP(r), UserAgent: r.UserAgent(), + AfterState: map[string]any{"status": item.Status, "attempts": item.Attempts}, + Metadata: map[string]any{"taskId": item.TaskID, "action": item.Action, "currency": item.Currency}, + } + if actor != nil { + input.ActorGatewayUserID = uuidText(firstNonEmptyText(actor.GatewayUserID, actor.ID)) + input.ActorUserID = actor.ID + input.ActorUsername = actor.Username + input.ActorSource = actor.Source + input.ActorRoles = actor.Roles + } + return input +} diff --git a/apps/api/internal/httpapi/billing_settlement_handlers_test.go b/apps/api/internal/httpapi/billing_settlement_handlers_test.go new file mode 100644 index 0000000..6d49445 --- /dev/null +++ b/apps/api/internal/httpapi/billing_settlement_handlers_test.go @@ -0,0 +1,21 @@ +package httpapi + +import ( + "net/http/httptest" + "testing" +) + +func TestSingleIdempotencyKey(t *testing.T) { + t.Parallel() + + request := httptest.NewRequest("POST", "/", nil) + request.Header.Add("Idempotency-Key", "retry-1") + if got, ok := singleIdempotencyKey(request); !ok || got != "retry-1" { + t.Fatalf("got key=%q ok=%v", got, ok) + } + + request.Header.Add("Idempotency-Key", "retry-2") + if _, ok := singleIdempotencyKey(request); ok { + t.Fatal("multiple Idempotency-Key values must be rejected") + } +} diff --git a/apps/api/internal/httpapi/core_flow_integration_test.go b/apps/api/internal/httpapi/core_flow_integration_test.go index 7baeb75..a400451 100644 --- a/apps/api/internal/httpapi/core_flow_integration_test.go +++ b/apps/api/internal/httpapi/core_flow_integration_test.go @@ -444,7 +444,6 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil { t.Fatalf("unexpected compatible chat response: %+v", compatChat) } - cancelMarker := "cancel-stream-" + suffixText cancelCtx, cancelRequest := context.WithCancel(context.Background()) cancelPayload := map[string]any{ "model": defaultTextModel, @@ -453,7 +452,6 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil { "stream": true, "simulation": true, "simulationDurationMs": 250, - "cancelTestId": cancelMarker, } cancelRaw, err := json.Marshal(cancelPayload) if err != nil { @@ -466,15 +464,27 @@ VALUES ($1, 5, '{"purpose":"core-flow"}'::jsonb)`, inviteCode); err != nil { cancelReq.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret) cancelReq.Header.Set("Content-Type", "application/json") cancelErrCh := make(chan error, 1) + cancelTaskIDCh := make(chan string, 1) go func() { resp, err := http.DefaultClient.Do(cancelReq) if resp != nil { + cancelTaskIDCh <- strings.TrimSpace(resp.Header.Get("X-Gateway-Task-Id")) _, _ = io.ReadAll(resp.Body) _ = resp.Body.Close() + } else { + cancelTaskIDCh <- "" } cancelErrCh <- err }() - cancelTaskID := waitForTaskIDByRequestMarker(t, ctx, testPool, cancelMarker, 2*time.Second) + var cancelTaskID string + select { + case cancelTaskID = <-cancelTaskIDCh: + case <-time.After(2 * time.Second): + t.Fatal("cancelled stream did not return response headers") + } + if cancelTaskID == "" { + t.Fatal("cancelled stream response did not expose X-Gateway-Task-Id") + } cancelRequest() select { case <-cancelErrCh: @@ -845,7 +855,7 @@ WHERE gateway_user_id = $1::uuid } doAPIV1ChatCompletionAndLoadTask(t, ctx, testPool, server.URL, apiKeyResponse.Secret, map[string]any{ "model": pricingModel, - "runMode": "simulation", + "runMode": "production", "simulation": true, "simulationDurationMs": 5, "messages": []map[string]any{{"role": "user", "content": "priced ping"}}, @@ -853,6 +863,20 @@ WHERE gateway_user_id = $1::uuid if pricingTask.Task.Status != "succeeded" || !floatNear(pricingTask.Task.FinalChargeAmount, 0.028) { t.Fatalf("custom pricing rule set should drive text billing, got task=%+v", pricingTask.Task) } + settlementDeadline := time.Now().Add(3 * time.Second) + for { + var billingStatus string + if err := testPool.QueryRow(ctx, `SELECT billing_status FROM gateway_tasks WHERE id = $1::uuid`, pricingTask.Task.ID).Scan(&billingStatus); err != nil { + t.Fatalf("read pricing task billing status: %v", err) + } + if billingStatus == "settled" { + break + } + if time.Now().After(settlementDeadline) { + t.Fatalf("pricing task billing did not settle before deadline, status=%s", billingStatus) + } + time.Sleep(25 * time.Millisecond) + } var walletBalanceAfter float64 var walletSpentAfter float64 if err := testPool.QueryRow(ctx, ` @@ -1484,7 +1508,7 @@ WHERE m.platform_id = $1::uuid t.Fatalf("workspace task list should include persisted task records, got %+v", workspaceTaskList.Items) } - req, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/tasks/"+taskResponse.Task.ID+"/events", nil) + req, err := http.NewRequest(http.MethodGet, server.URL+"/api/v1/tasks/"+pricingTask.Task.ID+"/events", nil) if err != nil { t.Fatalf("build events request: %v", err) } @@ -1864,7 +1888,7 @@ func doJSON(t *testing.T, baseURL string, method string, path string, token stri doJSONWithHeaders(t, baseURL, method, path, token, payload, nil, expectedStatus, out) } -func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string, token string, payload any, headers map[string]string, expectedStatus int, out any) { +func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string, token string, payload any, headers map[string]string, expectedStatus int, out any) http.Header { t.Helper() var body io.Reader if payload != nil { @@ -1901,16 +1925,22 @@ func doJSONWithHeaders(t *testing.T, baseURL string, method string, path string, t.Fatalf("decode %s %s response: %v body=%s", method, path, err, string(raw)) } } + return resp.Header.Clone() } func doAPIV1ChatCompletionAndLoadTask(t *testing.T, ctx context.Context, pool *pgxpool.Pool, baseURL string, token string, payload map[string]any, marker string, expectedStatus int, responseOut any, taskDetailOut any) string { t.Helper() - payload["integrationTestMarker"] = marker + _ = ctx + _ = pool + _ = marker if responseOut == nil { responseOut = &map[string]any{} } - doJSON(t, baseURL, http.MethodPost, "/api/v1/chat/completions", token, payload, expectedStatus, responseOut) - taskID := waitForTaskIDByRequestField(t, ctx, pool, "integrationTestMarker", marker, 2*time.Second) + responseHeaders := doJSONWithHeaders(t, baseURL, http.MethodPost, "/api/v1/chat/completions", token, payload, nil, expectedStatus, responseOut) + taskID := strings.TrimSpace(responseHeaders.Get("X-Gateway-Task-Id")) + if taskID == "" { + t.Fatal("chat completion response did not expose X-Gateway-Task-Id") + } if taskDetailOut != nil { doJSON(t, baseURL, http.MethodGet, "/api/v1/tasks/"+taskID, token, nil, http.StatusOK, taskDetailOut) } @@ -2046,11 +2076,6 @@ func waitForTaskStatus(t *testing.T, baseURL string, token string, taskID string return detail } -func waitForTaskIDByRequestMarker(t *testing.T, ctx context.Context, pool *pgxpool.Pool, marker string, timeout time.Duration) string { - t.Helper() - return waitForTaskIDByRequestField(t, ctx, pool, "cancelTestId", marker, timeout) -} - func waitForTaskIDByRequestField(t *testing.T, ctx context.Context, pool *pgxpool.Pool, key string, value string, timeout time.Duration) string { t.Helper() deadline := time.Now().Add(timeout) diff --git a/apps/api/internal/httpapi/gemini_compat.go b/apps/api/internal/httpapi/gemini_compat.go index 72834f3..16366fc 100644 --- a/apps/api/internal/httpapi/gemini_compat.go +++ b/apps/api/internal/httpapi/gemini_compat.go @@ -4,6 +4,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "mime" @@ -94,6 +95,11 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusForbidden, "api key scope does not allow this capability") return } + idempotencyKey, hasIdempotencyKey, err := optionalTaskIdempotencyKey(r) + if err != nil { + writeError(w, http.StatusBadRequest, "Idempotency-Key must contain one non-empty value", "invalid_idempotency_key") + return + } prepared, err := s.prepareTaskRequest(r.Context(), r, user, mapping.Body) if err != nil { s.logger.Warn("prepare gemini task request failed", "kind", mapping.Kind, "error", err) @@ -104,7 +110,7 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { writeError(w, status, err.Error(), clients.ErrorCode(err)) return } - task, err := s.store.CreateTask(r.Context(), store.CreateTaskInput{ + createInput := store.CreateTaskInput{ Kind: mapping.Kind, Model: mapping.Model, RunMode: runModeFromRequest(prepared.Body), @@ -113,10 +119,33 @@ func (s *Server) geminiGenerateContent(w http.ResponseWriter, r *http.Request) { ConversationID: prepared.ConversationID, NewMessageCount: prepared.NewMessageCount, MessageRefs: prepared.MessageRefs, - }, user) + } + if hasIdempotencyKey { + createInput.IdempotencyKeyHash = taskIdempotencyKeyHash(idempotencyKey) + createInput.IdempotencyRequestHash = taskIdempotencyRequestHash(mapping.Kind, false, false, prepared.Body) + } + created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user) if err != nil { - s.logger.Error("create gemini task failed", "kind", mapping.Kind, "error", err) - writeError(w, http.StatusInternalServerError, "create task failed") + if errors.Is(err, store.ErrIdempotencyKeyReused) { + writeError(w, http.StatusConflict, "Idempotency-Key was reused for a different request", "idempotency_key_reused") + return + } + s.logger.Error("create gemini task failed", "kind", mapping.Kind, "error_category", "task_create_failed") + writeError(w, http.StatusInternalServerError, "create task failed", "task_create_failed") + return + } + task := created.Task + if created.Replayed { + if s.billingMetrics != nil { + s.billingMetrics.ObserveBillingEvent("idempotent_replay") + } + w.Header().Set("Idempotent-Replayed", "true") + w.Header().Set("X-Gateway-Task-Id", task.ID) + if task.Status == "succeeded" { + writeJSON(w, http.StatusOK, geminiGenerateContentResponse(task.Result, mapping.Model)) + return + } + writeIdempotentTaskReplay(w, task, true) return } runCtx, cancelRun := s.requestExecutionContext(r) diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 2a45dad..03dbae9 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -917,7 +917,13 @@ func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) { } estimate, err := s.runner.Estimate(r.Context(), kind, model, body, user) if err != nil { + if s.billingMetrics != nil { + s.billingMetrics.ObserveBillingEvent("estimate_failed") + } if runner.IsPricingUnavailable(err) { + if s.billingMetrics != nil { + s.billingMetrics.ObserveBillingEvent("pricing_unavailable") + } writeErrorWithDetails(w, http.StatusServiceUnavailable, runErrorMessage(err), runErrorDetails(err), "pricing_unavailable") return } @@ -982,6 +988,7 @@ func (s *Server) listModelRateLimitStatuses(w http.ResponseWriter, r *http.Reque // @Produce json // @Security BearerAuth // @Param X-Async header bool false "true 时异步创建任务并返回 202" +// @Param Idempotency-Key header string false "可选请求幂等键；同一用户范围内唯一" // @Param input body TaskRequest true "AI 任务请求，字段随任务类型变化" // @Success 200 {object} CompatibleResponse // @Success 202 {object} TaskAcceptedResponse @@ -1050,6 +1057,11 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { return } responsePlan := planTaskResponse(kind, compatible, body, r) + idempotencyKey, hasIdempotencyKey, err := optionalTaskIdempotencyKey(r) + if err != nil { + writeError(w, http.StatusBadRequest, "Idempotency-Key must contain one non-empty value", "invalid_idempotency_key") + return + } prepared, err := s.prepareTaskRequest(r.Context(), r, user, body) if err != nil { s.logger.Warn("prepare task request failed", "kind", kind, "error", err) @@ -1061,7 +1073,7 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { return } - task, err := s.store.CreateTask(r.Context(), store.CreateTaskInput{ + createInput := store.CreateTaskInput{ Kind: kind, Model: model, RunMode: runModeFromRequest(prepared.Body), @@ -1070,10 +1082,32 @@ func (s *Server) createTask(kind string, compatible bool) http.Handler { ConversationID: prepared.ConversationID, NewMessageCount: prepared.NewMessageCount, MessageRefs: prepared.MessageRefs, - }, user) + } + if hasIdempotencyKey { + createInput.IdempotencyKeyHash = taskIdempotencyKeyHash(idempotencyKey) + createInput.IdempotencyRequestHash = taskIdempotencyRequestHash(kind, responsePlan.asyncMode, responsePlan.streamMode, prepared.Body) + } + created, err := s.store.CreateTaskIdempotent(r.Context(), createInput, user) if err != nil { - s.logger.Error("create task failed", "kind", kind, "error", err) - writeError(w, http.StatusInternalServerError, "create task failed") + if errors.Is(err, store.ErrIdempotencyKeyReused) { + writeError(w, http.StatusConflict, "Idempotency-Key was reused for a different request", "idempotency_key_reused") + return + } + s.logger.Error("create task failed", "kind", kind, "error_category", "task_create_failed") + writeError(w, http.StatusInternalServerError, "create task failed", "task_create_failed") + return + } + task := created.Task + if created.Replayed { + if s.billingMetrics != nil { + s.billingMetrics.ObserveBillingEvent("idempotent_replay") + } + w.Header().Set("Idempotent-Replayed", "true") + if responsePlan.streamMode { + writeError(w, http.StatusConflict, "streaming idempotent replay is not supported", "idempotency_stream_replay_unsupported") + return + } + writeIdempotentTaskReplay(w, task, responsePlan.compatibleMode) return } if responsePlan.asyncMode { @@ -1388,6 +1422,8 @@ func scopeForTaskKind(kind string) string { func statusFromRunError(err error) int { switch { + case clients.ErrorCode(err) == "billing_hold": + return http.StatusServiceUnavailable case runner.IsPricingUnavailable(err): return http.StatusServiceUnavailable case clients.ErrorCode(err) == "invalid_previous_response_id" || clients.ErrorCode(err) == "response_chain_too_deep" || clients.ErrorCode(err) == "unsupported_model_protocol" || clients.ErrorCode(err) == "unsupported_response_tool" || clients.ErrorCode(err) == "unsupported_response_parameter": diff --git a/apps/api/internal/httpapi/oidc_jit_integration_test.go b/apps/api/internal/httpapi/oidc_jit_integration_test.go index d4fd8ad..d80f831 100644 --- a/apps/api/internal/httpapi/oidc_jit_integration_test.go +++ b/apps/api/internal/httpapi/oidc_jit_integration_test.go @@ -43,7 +43,7 @@ func TestOIDCJITFullGatewayUserFlowAndSecurityBoundary(t *testing.T) { if err != nil { t.Fatalf("connect store: %v", err) } - defer db.Close() + t.Cleanup(db.Close) key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { @@ -247,6 +247,9 @@ DELETE FROM gateway_users WHERE source = 'oidc' AND external_user_id = ANY($1::t t.Fatalf("rejected OIDC tokens created %d local users", rejectedWrites) } + if _, err := db.DisableActiveIdentityRevision(ctx, activeRevision.Version, "oidc-jit-disabled", "oidc-jit-disabled"); err != nil { + t.Fatalf("disable initial OIDC JIT revision: %v", err) + } disabledJITRevision := prepareOIDCJITRevision(t, ctx, db, baseConfig, issuer, "default", false) testRevisionIDs = append(testRevisionIDs, disabledJITRevision.ID) disabledJITRevision, _, err = db.ActivateIdentityRevision(ctx, disabledJITRevision.ID, disabledJITRevision.Version, "oidc-jit-disabled", "oidc-jit-disabled") @@ -298,6 +301,9 @@ func prepareOIDCJITRevision(t *testing.T, ctx context.Context, db *store.Store, if err := secrets.Put(ctx, sessionReference, bytes.Repeat([]byte{8}, 32)); err != nil { t.Fatalf("store OIDC JIT session key: %v", err) } + if err := db.QueueIdentitySecretCleanup(ctx, sessionReference, time.Now().Add(10*time.Minute)); err != nil { + t.Fatalf("stage OIDC JIT session key for adoption: %v", err) + } draft, err = db.ApplyIdentityManifest(ctx, draft.ID, draft.Version, identity.ManifestApplication{ Manifest: identity.ManifestV1{ SchemaVersion: 1, Issuer: issuer, TenantID: oidcJITTenantID, ApplicationID: uuid.NewString(), @@ -360,6 +366,10 @@ var currentOIDCTestNonce string func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie { t.Helper() + parsedBaseURL, err := url.Parse(baseURL) + if err != nil { + t.Fatal(err) + } jar, err := cookiejar.New(nil) if err != nil { t.Fatal(err) @@ -371,6 +381,11 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie { if len(via) > 10 { return errors.New("too many redirects") } + // Follow the synthetic issuer's authorization redirect, then stop before + // the callback redirects the browser to the separately hosted web app. + if request.URL.Host != parsedBaseURL.Host && request.URL.Path != "/authorize" { + return http.ErrUseLastResponse + } return nil }} response, err := client.Get(baseURL + "/api/v1/auth/oidc/login?returnTo=%2Fapi%2Fv1%2Fme") @@ -378,11 +393,10 @@ func createOIDCBFFSessionCookie(t *testing.T, baseURL string) *http.Cookie { t.Fatalf("complete OIDC BFF login: %v", err) } defer response.Body.Close() - if response.StatusCode != http.StatusOK { + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusSeeOther { body, _ := io.ReadAll(response.Body) t.Fatalf("OIDC BFF login status = %d, want 200: %s", response.StatusCode, body) } - parsedBaseURL, _ := url.Parse(baseURL) for _, cookie := range jar.Cookies(parsedBaseURL) { if cookie.Name == auth.OIDCSessionCookieName { if strings.Count(cookie.Value, ".") == 2 { diff --git a/apps/api/internal/httpapi/openapi_models.go b/apps/api/internal/httpapi/openapi_models.go index 59bad83..19a922d 100644 --- a/apps/api/internal/httpapi/openapi_models.go +++ b/apps/api/internal/httpapi/openapi_models.go @@ -114,6 +114,18 @@ type AuditLogListResponse struct { Items []store.AuditLog `json:"items"` } +type BillingSettlementListResponse struct { + Items []store.BillingSettlement `json:"items"` + Total int `json:"total" example:"42"` + Page int `json:"page" example:"1"` + PageSize int `json:"pageSize" example:"50"` +} + +type BillingSettlementRetryResponse struct { + Settlement store.BillingSettlement `json:"settlement"` + AuditLog store.AuditLog `json:"auditLog"` +} + type WalletTransactionListResponse struct { Items []store.GatewayWalletTransaction `json:"items"` Total int `json:"total" example:"42"` diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index ab2bd16..4dc9020 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -43,6 +43,7 @@ type Server struct { identityTestRevision identity.Revision identityTestCookieSecure bool identityTestBrowserEnabled bool + billingMetrics *ssfreceiver.Metrics } type oidcPublicClient interface { @@ -66,18 +67,19 @@ func NewServer(cfg config.Config, db *store.Store, logger *slog.Logger) http.Han } func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Store, logger *slog.Logger) http.Handler { + securityEventMetrics := &ssfreceiver.Metrics{} server := &Server{ ctx: ctx, cfg: cfg, store: db, oidcUserResolver: db, auth: auth.New(cfg.JWTSecret, cfg.ServerMainBaseURL, cfg.ServerMainInternalToken), - runner: runner.New(cfg, db, logger), + runner: runner.New(cfg, db, logger, securityEventMetrics), logger: logger, + billingMetrics: securityEventMetrics, } server.auth.ServerMainInternalKey = cfg.ServerMainInternalKey server.auth.ServerMainInternalSecret = cfg.ServerMainInternalSecret - securityEventMetrics := &ssfreceiver.Metrics{} secretStore, err := identitySecretStore(cfg) if err != nil { panic("invalid identity SecretStore: " + err.Error()) @@ -118,6 +120,7 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor } server.auth.LocalAPIKeyVerifier = db.VerifyLocalAPIKey server.runner.StartAsyncQueueWorker(ctx) + server.runner.StartBillingSettlementWorker(ctx) server.startLocalTempAssetCleanup(ctx) server.startOIDCSessionCleanup(ctx) @@ -204,6 +207,8 @@ func NewServerWithContext(ctx context.Context, cfg config.Config, db *store.Stor mux.Handle("DELETE /api/admin/runtime/policy-sets/{policySetID}", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.deleteRuntimePolicySet))) mux.Handle("GET /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getRunnerPolicy))) mux.Handle("PATCH /api/admin/runtime/runner-policy", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.updateRunnerPolicy))) + mux.Handle("GET /api/admin/runtime/billing-settlements", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.listBillingSettlements))) + mux.Handle("POST /api/admin/runtime/billing-settlements/{settlementId}/retry", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.retryBillingSettlement))) mux.Handle("POST /api/admin/runtime/model-rate-limits/{platformModelID}/restore", server.requireAdmin(auth.PermissionManager, http.HandlerFunc(server.restorePlatformModelRuntimeStatus))) mux.Handle("GET /api/admin/config/network-proxy", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getNetworkProxyConfig))) mux.Handle("GET /api/admin/system/file-storage/settings", server.requireAdmin(auth.PermissionPower, http.HandlerFunc(server.getFileStorageSettings))) diff --git a/apps/api/internal/httpapi/task_idempotency.go b/apps/api/internal/httpapi/task_idempotency.go new file mode 100644 index 0000000..467b7e1 --- /dev/null +++ b/apps/api/internal/httpapi/task_idempotency.go @@ -0,0 +1,85 @@ +package httpapi + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/http" + "strings" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +var errInvalidTaskIdempotencyKey = errors.New("invalid Idempotency-Key header") + +func optionalTaskIdempotencyKey(r *http.Request) (string, bool, error) { + values := r.Header.Values("Idempotency-Key") + if len(values) == 0 { + return "", false, nil + } + if len(values) != 1 { + return "", false, errInvalidTaskIdempotencyKey + } + value := strings.TrimSpace(values[0]) + if value == "" || len(value) > 255 || strings.Contains(value, ",") { + return "", false, errInvalidTaskIdempotencyKey + } + return value, true, nil +} + +func taskIdempotencyKeyHash(key string) string { + digest := sha256.Sum256([]byte(key)) + return hex.EncodeToString(digest[:]) +} + +func taskIdempotencyRequestHash(kind string, async bool, stream bool, body map[string]any) string { + payload, _ := json.Marshal(map[string]any{ + "kind": kind, "async": async, "stream": stream, "request": body, + }) + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func writeIdempotentTaskReplay(w http.ResponseWriter, task store.GatewayTask, compatible bool) { + w.Header().Set("Idempotent-Replayed", "true") + w.Header().Set("X-Gateway-Task-Id", task.ID) + if !compatible || task.AsyncMode || (task.Status != "succeeded" && task.Status != "failed" && task.Status != "cancelled") { + writeTaskAccepted(w, task) + return + } + if task.Status == "succeeded" { + writeJSON(w, http.StatusOK, task.Result) + return + } + status := storedTaskErrorStatus(task.ErrorCode) + message := strings.TrimSpace(task.ErrorMessage) + if message == "" { + message = strings.TrimSpace(task.Error) + } + if message == "" { + message = "task failed" + } + code := strings.TrimSpace(task.ErrorCode) + if code == "" { + code = "task_failed" + } + writeError(w, status, message, code) +} + +func storedTaskErrorStatus(code string) int { + switch strings.TrimSpace(code) { + case "pricing_unavailable", "response_chain_unavailable", "billing_hold": + return http.StatusServiceUnavailable + case "insufficient_balance": + return http.StatusPaymentRequired + case "bad_request", "invalid_parameter", "invalid_previous_response_id", "unsupported_operation": + return http.StatusBadRequest + case "no_model_candidate", "cloned_voice_not_found": + return http.StatusNotFound + case "rate_limit", "platform_cooling_down", "model_cooling_down": + return http.StatusTooManyRequests + default: + return http.StatusBadGateway + } +} diff --git a/apps/api/internal/httpapi/task_idempotency_test.go b/apps/api/internal/httpapi/task_idempotency_test.go new file mode 100644 index 0000000..4d49609 --- /dev/null +++ b/apps/api/internal/httpapi/task_idempotency_test.go @@ -0,0 +1,31 @@ +package httpapi + +import ( + "net/http/httptest" + "testing" +) + +func TestOptionalTaskIdempotencyKeyRejectsMultipleValues(t *testing.T) { + t.Parallel() + request := httptest.NewRequest("POST", "/", nil) + if _, present, err := optionalTaskIdempotencyKey(request); err != nil || present { + t.Fatalf("missing key present=%v err=%v", present, err) + } + request.Header.Add("Idempotency-Key", "one") + request.Header.Add("Idempotency-Key", "two") + if _, _, err := optionalTaskIdempotencyKey(request); err == nil { + t.Fatal("multiple keys must be rejected") + } +} + +func TestTaskIdempotencyRequestHashIsCanonical(t *testing.T) { + t.Parallel() + first := map[string]any{"model": "m", "n": float64(1), "nested": map[string]any{"b": true, "a": "x"}} + second := map[string]any{"nested": map[string]any{"a": "x", "b": true}, "n": float64(1), "model": "m"} + if taskIdempotencyRequestHash("images.generations", false, false, first) != taskIdempotencyRequestHash("images.generations", false, false, second) { + t.Fatal("equivalent JSON objects must have the same request hash") + } + if taskIdempotencyRequestHash("images.generations", true, false, first) == taskIdempotencyRequestHash("images.generations", false, false, first) { + t.Fatal("async response semantics must be part of the request hash") + } +} diff --git a/apps/api/internal/runner/billing_mode_test.go b/apps/api/internal/runner/billing_mode_test.go new file mode 100644 index 0000000..06cfd44 --- /dev/null +++ b/apps/api/internal/runner/billing_mode_test.go @@ -0,0 +1,27 @@ +package runner + +import "testing" + +func TestNormalizedBillingEngineMode(t *testing.T) { + t.Parallel() + if got := normalizedBillingEngineMode(""); got != "observe" { + t.Fatalf("empty mode = %q", got) + } + if got := normalizedBillingEngineMode("ENFORCE"); got != "enforce" { + t.Fatalf("enforce mode = %q", got) + } + if got := normalizedBillingEngineMode("hold"); got != "hold" { + t.Fatalf("hold mode = %q", got) + } +} + +func TestBillingItemsFixedTotalKeepsNineDecimalPlaces(t *testing.T) { + t.Parallel() + items := []any{ + map[string]any{"amount": "0.000000001"}, + map[string]any{"amount": float64(0.000000002)}, + } + if got := billingItemsFixedTotal(items).String(); got != "0.000000003" { + t.Fatalf("total = %s", got) + } +} diff --git a/apps/api/internal/runner/billing_settlement_worker.go b/apps/api/internal/runner/billing_settlement_worker.go new file mode 100644 index 0000000..20830f5 --- /dev/null +++ b/apps/api/internal/runner/billing_settlement_worker.go @@ -0,0 +1,96 @@ +package runner + +import ( + "context" + "errors" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" +) + +const billingSettlementPollInterval = time.Second + +func billingSettlementRetryDelay(attempt int) time.Duration { + if attempt < 1 { + attempt = 1 + } + delay := time.Second + for current := 1; current < attempt && delay < 15*time.Minute; current++ { + delay *= 2 + } + if delay > 15*time.Minute { + return 15 * time.Minute + } + return delay +} + +func billingSettlementErrorCode(err error) string { + if errors.Is(err, store.ErrInsufficientWalletBalance) { + return "insufficient_balance" + } + return "settlement_failed" +} + +func billingSettlementErrorMessage(code string) string { + if code == "insufficient_balance" { + return "wallet balance is insufficient for settlement" + } + return "billing settlement processing failed" +} + +func (s *Service) StartBillingSettlementWorker(ctx context.Context) { + workerID := "billing-" + uuid.NewString() + go s.runBillingSettlementWorker(ctx, workerID) +} + +func (s *Service) runBillingSettlementWorker(ctx context.Context, workerID string) { + ticker := time.NewTicker(billingSettlementPollInterval) + defer ticker.Stop() + + for { + s.processBillingSettlementBatch(ctx, workerID) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (s *Service) processBillingSettlementBatch(ctx context.Context, workerID string) { + items, err := s.store.ClaimBillingSettlements(ctx, workerID, store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout) + if err != nil { + if ctx.Err() == nil { + s.logger.Error("claim billing settlements failed", "error_category", "billing_settlement_claim_failed") + } + return + } + for _, item := range items { + if ctx.Err() != nil { + return + } + if err := s.store.ProcessBillingSettlement(ctx, item); err != nil { + code := billingSettlementErrorCode(err) + markErr := s.store.MarkBillingSettlementFailed( + context.WithoutCancel(ctx), + item, + code, + billingSettlementErrorMessage(code), + billingSettlementRetryDelay(item.Attempts), + ) + if markErr != nil { + s.logger.Error("mark billing settlement failed", "settlementID", item.ID, "taskID", item.TaskID, "error_category", "billing_settlement_state_failed") + continue + } + s.observeBillingEvent("settlement_retry") + if item.Attempts >= store.BillingSettlementMaxAttempts { + s.observeBillingEvent("manual_review") + } + s.logger.Warn("billing settlement scheduled for retry", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action, "error_category", code, "attempt", item.Attempts) + continue + } + s.observeBillingEvent("settlement_completed") + s.logger.Debug("billing settlement completed", "settlementID", item.ID, "taskID", item.TaskID, "action", item.Action) + } +} diff --git a/apps/api/internal/runner/billing_settlement_worker_test.go b/apps/api/internal/runner/billing_settlement_worker_test.go new file mode 100644 index 0000000..a02c17d --- /dev/null +++ b/apps/api/internal/runner/billing_settlement_worker_test.go @@ -0,0 +1,40 @@ +package runner + +import ( + "errors" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" +) + +func TestBillingSettlementRetryDelay(t *testing.T) { + t.Parallel() + + tests := []struct { + attempt int + want time.Duration + }{ + {attempt: 1, want: time.Second}, + {attempt: 2, want: 2 * time.Second}, + {attempt: 10, want: 512 * time.Second}, + {attempt: 11, want: 15 * time.Minute}, + {attempt: 20, want: 15 * time.Minute}, + } + for _, test := range tests { + if got := billingSettlementRetryDelay(test.attempt); got != test.want { + t.Fatalf("attempt %d: got %s, want %s", test.attempt, got, test.want) + } + } +} + +func TestBillingSettlementErrorCode(t *testing.T) { + t.Parallel() + + if got := billingSettlementErrorCode(store.ErrInsufficientWalletBalance); got != "insufficient_balance" { + t.Fatalf("got %q", got) + } + if got := billingSettlementErrorCode(errors.New("boom")); got != "settlement_failed" { + t.Fatalf("got %q", got) + } +} diff --git a/apps/api/internal/runner/execution_lease.go b/apps/api/internal/runner/execution_lease.go new file mode 100644 index 0000000..58e21de --- /dev/null +++ b/apps/api/internal/runner/execution_lease.go @@ -0,0 +1,28 @@ +package runner + +import ( + "context" + "time" +) + +const ( + taskExecutionLeaseTTL = 5 * time.Minute + taskExecutionRenewInterval = 30 * time.Second +) + +func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.CancelFunc, taskID string, executionToken string) { + ticker := time.NewTicker(taskExecutionRenewInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := s.store.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil { + s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost") + cancel() + return + } + } + } +} diff --git a/apps/api/internal/runner/pricing.go b/apps/api/internal/runner/pricing.go index 901a866..fb166d5 100644 --- a/apps/api/internal/runner/pricing.go +++ b/apps/api/internal/runner/pricing.go @@ -37,19 +37,14 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body return EstimateResult{}, err } estimates := make([]candidateEstimate, 0, len(candidates)) - var pricingErr error for _, candidate := range candidates { candidateBody := preprocessRequest(kind, cloneMap(body), candidate) estimate, candidateErr := s.estimateCandidateV2(ctx, user, kind, candidateBody, candidate) if candidateErr != nil { - pricingErr = candidateErr - continue + return EstimateResult{}, candidateErr } estimates = append(estimates, estimate) } - if len(estimates) == 0 && pricingErr != nil { - return EstimateResult{}, pricingErr - } return buildEstimateResult(estimates, pricingRequestFingerprint(kind, model, body)) } diff --git a/apps/api/internal/runner/queue_worker.go b/apps/api/internal/runner/queue_worker.go index 2b3e8b7..17880b8 100644 --- a/apps/api/internal/runner/queue_worker.go +++ b/apps/api/internal/runner/queue_worker.go @@ -10,6 +10,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" "github.com/riverqueue/river" "github.com/riverqueue/river/riverdriver/riverpgxv5" "github.com/riverqueue/river/rivermigrate" @@ -38,16 +39,22 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs if task.Status == "succeeded" || task.Status == "failed" || task.Status == "cancelled" { return nil } - result, runErr := w.service.Execute(ctx, task, authUserFromTask(task)) + executionToken := uuid.NewString() + result, runErr := w.service.executeWithToken(ctx, task, authUserFromTask(task), nil, executionToken) if runErr == nil { w.service.logger.Debug("river async task completed", "taskID", task.ID, "status", result.Task.Status, "riverJobID", job.ID) return nil } + if errors.Is(runErr, store.ErrTaskExecutionLeaseUnavailable) { + w.service.logger.Debug("river async task execution lease already held", "taskID", task.ID, "riverJobID", job.ID) + return nil + } var queuedErr *TaskQueuedError if errors.As(runErr, &queuedErr) { return river.JobSnooze(queuedErr.Delay) } if ctx.Err() != nil { + task.ExecutionToken = executionToken queued, queueErr := w.service.requeueInterruptedAsyncTask(context.WithoutCancel(ctx), task) if queueErr != nil { return queueErr @@ -145,8 +152,7 @@ func (s *Service) recoverAsyncRiverJobs(ctx context.Context) error { return err } for _, item := range items { - task := store.GatewayTask{ID: item.ID} - result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskInsertOpts(task)) + result, err := s.riverClient.Insert(ctx, asyncTaskArgs{TaskID: item.ID}, asyncTaskRecoveryInsertOpts(item)) if err != nil { return err } @@ -186,6 +192,16 @@ func asyncTaskInsertOpts(task store.GatewayTask) *river.InsertOpts { } } +func asyncTaskRecoveryInsertOpts(item store.AsyncTaskQueueItem) *river.InsertOpts { + opts := asyncTaskInsertOpts(store.GatewayTask{ID: item.ID}) + opts.ScheduledAt = item.NextRunAt + // A replacement process must not be blocked by a River row that the dead + // process left in running state. PostgreSQL execution leases still ensure + // that only one recovery job can call the upstream provider. + opts.UniqueOpts = river.UniqueOpts{} + return opts +} + func authUserFromTask(task store.GatewayTask) *auth.User { roles := []string{"user"} if strings.TrimSpace(task.UserID) == "" { diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 3ab59e1..2f2b29f 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -14,6 +14,7 @@ import ( "github.com/easyai/easyai-ai-gateway/apps/api/internal/config" scriptengine "github.com/easyai/easyai-ai-gateway/apps/api/internal/script" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/riverqueue/river" ) @@ -26,6 +27,11 @@ type Service struct { scriptExecutor *scriptengine.Executor httpClients *httpClientCache riverClient *river.Client[pgx.Tx] + billingMetrics billingMetricsObserver +} + +type billingMetricsObserver interface { + ObserveBillingEvent(string) } type Result struct { @@ -39,6 +45,19 @@ type TaskQueuedError struct { Delay time.Duration } +type upstreamSubmissionUnknownError struct { + AttemptID string + Cause error +} + +func (e *upstreamSubmissionUnknownError) Error() string { + return "upstream submission result is unknown" +} + +func (e *upstreamSubmissionUnknownError) Unwrap() error { + return e.Cause +} + func (e *TaskQueuedError) Error() string { return ErrTaskQueued.Error() } @@ -47,10 +66,10 @@ func (e *TaskQueuedError) Is(target error) bool { return target == ErrTaskQueued } -func New(cfg config.Config, db *store.Store, logger *slog.Logger) *Service { +func New(cfg config.Config, db *store.Store, logger *slog.Logger, observers ...billingMetricsObserver) *Service { httpClients := newHTTPClientCache() scriptExecutor := &scriptengine.Executor{Logger: logger} - return &Service{ + service := &Service{ cfg: cfg, store: db, logger: logger, @@ -76,6 +95,16 @@ func New(cfg config.Config, db *store.Store, logger *slog.Logger) *Service { }, httpClients: httpClients, } + if len(observers) > 0 { + service.billingMetrics = observers[0] + } + return service +} + +func (s *Service) observeBillingEvent(event string) { + if s.billingMetrics != nil { + s.billingMetrics.ObserveBillingEvent(event) + } } func (s *Service) Execute(ctx context.Context, task store.GatewayTask, user *auth.User) (Result, error) { @@ -87,6 +116,20 @@ func (s *Service) ExecuteStream(ctx context.Context, task store.GatewayTask, use } func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta) (Result, error) { + return s.executeWithToken(ctx, task, user, onDelta, uuid.NewString()) +} + +func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, user *auth.User, onDelta clients.StreamDelta, executionToken string) (Result, error) { + wasRunning := task.Status == "running" + claimed, err := s.store.ClaimTaskExecution(ctx, task.ID, executionToken, taskExecutionLeaseTTL) + if err != nil { + return Result{}, err + } + task = claimed + executionCtx, stopExecution := context.WithCancel(ctx) + defer stopExecution() + go s.renewTaskExecutionLease(executionCtx, stopExecution, task.ID, task.ExecutionToken) + ctx = executionCtx executeStartedAt := time.Now() restoredRequest, err := s.restoreTaskRequestReferences(ctx, task) if err != nil { @@ -95,10 +138,10 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut body := normalizeRequest(task.Kind, restoredRequest) responseExecution := responseExecutionContext{} modelType := modelTypeFromKind(task.Kind, body) - if err := s.store.MarkTaskRunning(ctx, task.ID, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { + if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { return Result{}, err } - if task.Status != "running" { + if !wasRunning { if err := s.emit(ctx, task.ID, "task.running", "running", "starting", 0.12, "task pulled from queue and started", map[string]any{"modelType": modelType}, task.RunMode == "simulation"); err != nil { return Result{}, err } @@ -115,7 +158,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Reason: "request_validation_failed", ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, "bad_request", err.Error(), task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "bad_request", err.Error(), task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } @@ -135,14 +178,14 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Reason: "cloned_voice_binding_failed", ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(err), err.Error(), task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } return Result{Task: failed, Output: failed.Result}, err } if clonedVoice.Found { - if err := s.store.MarkTaskRunning(ctx, task.ID, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { + if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, modelType, s.slimTaskRequestSnapshot(task, body)); err != nil { return Result{}, err } } @@ -151,7 +194,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut if err != nil { code, message := responseExecutionFailure(err) s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_chain", Reason: code, ModelType: modelType}) - failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } @@ -183,7 +226,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Reason: "candidate_selection_failed", ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } @@ -202,7 +245,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Reason: store.ModelCandidateErrorCode(err), ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } @@ -224,7 +267,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut ExtraMetrics: []map[string]any{candidateFilterMetrics}, ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) if finishErr != nil { return Result{}, finishErr } @@ -240,7 +283,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Simulated: task.RunMode == "simulation", Scope: "candidate_output_token_filter", Reason: store.ModelCandidateErrorCode(err), ExtraMetrics: []map[string]any{candidateFilterMetrics}, ModelType: modelType, }) - failed, finishErr := s.failTask(ctx, task.ID, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, store.ModelCandidateErrorCode(err), err.Error(), task.RunMode == "simulation", err, candidateFilterMetrics) if finishErr != nil { return Result{}, finishErr } @@ -251,7 +294,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut if err != nil { code, message := responseExecutionFailure(err) s.recordFailedAttempt(ctx, failedAttemptRecord{Task: task, Body: body, AttemptNo: task.AttemptCount + 1, Code: code, Cause: err, Simulated: task.RunMode == "simulation", Scope: "response_protocol", Reason: code, ModelType: modelType}) - failed, finishErr := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", err) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", err) if finishErr != nil { return Result{}, finishErr } @@ -260,8 +303,17 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut } pricingByCandidate := map[string]resolvedPricing{} reservationBillings := []any(nil) + reservationPricingSnapshot := map[string]any(nil) if task.RunMode == "production" { - pricedCandidates := make([]store.RuntimeModelCandidate, 0, len(candidates)) + billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode) + if billingMode == "hold" { + holdErr := &clients.ClientError{Code: "billing_hold", Message: "production billing is temporarily on hold", Retryable: true} + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "billing_hold", holdErr.Error(), false, holdErr) + if finishErr != nil { + return Result{}, finishErr + } + return Result{Task: failed, Output: failed.Result}, holdErr + } estimates := make([]candidateEstimate, 0, len(candidates)) var pricingErr error for _, candidate := range candidates { @@ -269,32 +321,68 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut estimate, estimateErr := s.estimateCandidateV2(ctx, user, task.Kind, pricingBody, candidate) if estimateErr != nil { pricingErr = estimateErr + if billingMode == "enforce" { + break + } continue } - pricedCandidates = append(pricedCandidates, candidate) estimates = append(estimates, estimate) pricingByCandidate[pricingCandidateKey(candidate)] = estimate.Pricing } - if len(pricedCandidates) == 0 { - if pricingErr == nil { - pricingErr = &PricingUnavailableError{Reason: "no candidate has effective pricing"} + if billingMode == "observe" { + legacyItems, legacyAmount := s.maximumLegacyCandidateEstimate(ctx, user, task.Kind, body, candidates) + reservationBillings = legacyItems + candidateSnapshots := make([]any, 0, len(estimates)) + for _, estimate := range estimates { + candidateSnapshots = append(candidateSnapshots, estimate.Snapshot) } - failed, finishErr := s.failTask(ctx, task.ID, "pricing_unavailable", pricingErr.Error(), false, pricingErr) + observedAmount := "" + if maximumEstimate, estimateErr := maximumCandidateEstimate(estimates); estimateErr == nil { + observedAmount = maximumEstimate.Amount.String() + } + reservationPricingSnapshot = map[string]any{ + "pricingVersion": "legacy-observe", "observedPricingVersion": pricingVersionV2, + "requestFingerprint": pricingRequestFingerprint(task.Kind, task.Model, body), + "reservationAmount": legacyAmount.String(), "observedReservationAmount": observedAmount, + "candidateCount": len(candidates), "observedCandidates": candidateSnapshots, + } + s.logger.Info("billing observe comparison", "taskID", task.ID, "legacyAmount", legacyAmount.String(), "v2Amount", observedAmount, "pricedCandidates", len(estimates), "candidateCount", len(candidates)) + } else if pricingErr != nil || len(estimates) != len(candidates) { + if pricingErr == nil { + pricingErr = &PricingUnavailableError{Reason: "not every candidate has effective pricing"} + } + s.observeBillingEvent("pricing_unavailable") + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "pricing_unavailable", pricingErr.Error(), false, pricingErr) if finishErr != nil { return Result{}, finishErr } return Result{Task: failed, Output: failed.Result}, pricingErr - } - maximumEstimate, estimateErr := maximumCandidateEstimate(estimates) - if estimateErr != nil { - failed, finishErr := s.failTask(ctx, task.ID, "pricing_unavailable", estimateErr.Error(), false, estimateErr) - if finishErr != nil { - return Result{}, finishErr + } else { + maximumEstimate, estimateErr := maximumCandidateEstimate(estimates) + if estimateErr != nil { + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "pricing_unavailable", estimateErr.Error(), false, estimateErr) + if finishErr != nil { + return Result{}, finishErr + } + return Result{Task: failed, Output: failed.Result}, estimateErr + } + pricedCandidates := make([]store.RuntimeModelCandidate, 0, len(estimates)) + candidateSnapshots := make([]any, 0, len(estimates)) + for _, candidate := range candidates { + if _, ok := pricingByCandidate[pricingCandidateKey(candidate)]; ok { + pricedCandidates = append(pricedCandidates, candidate) + } + } + for _, estimate := range estimates { + candidateSnapshots = append(candidateSnapshots, estimate.Snapshot) + } + candidates = pricedCandidates + reservationBillings = maximumEstimate.Items + reservationPricingSnapshot = map[string]any{ + "pricingVersion": pricingVersionV2, "requestFingerprint": pricingRequestFingerprint(task.Kind, task.Model, body), + "reservationAmount": maximumEstimate.Amount.String(), "candidateCount": len(estimates), "candidates": candidateSnapshots, } - return Result{Task: failed, Output: failed.Result}, estimateErr } - candidates = pricedCandidates - reservationBillings = maximumEstimate.Items } firstCandidateBody := body normalizedModelType := modelType @@ -328,17 +416,17 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Preprocessing: &firstPreprocessing, ModelType: normalizedModelType, }) - failed, finishErr := s.failTask(ctx, task.ID, clients.ErrorCode(clientErr), clientErr.Error(), task.RunMode == "simulation", clientErr, parameterPreprocessingMetrics(firstPreprocessing)) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(clientErr), clientErr.Error(), task.RunMode == "simulation", clientErr, parameterPreprocessingMetrics(firstPreprocessing)) if finishErr != nil { return Result{}, finishErr } return Result{Task: failed, Output: failed.Result}, clientErr } - if err := s.store.MarkTaskRunning(ctx, task.ID, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil { + if err := s.store.MarkTaskRunning(ctx, task.ID, task.ExecutionToken, candidates[0].ModelType, s.slimTaskRequestSnapshot(task, firstCandidateBody)); err != nil { return Result{}, err } var reserveErr error - walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings) + walletReservations, reserveErr = s.store.ReserveTaskBilling(ctx, task, user, reservationBillings, reservationPricingSnapshot) if reserveErr != nil { if errors.Is(reserveErr, store.ErrInsufficientWalletBalance) { attemptNo = s.recordFailedAttempt(ctx, failedAttemptRecord{ @@ -355,7 +443,7 @@ func (s *Service) execute(ctx context.Context, task store.GatewayTask, user *aut Preprocessing: &firstPreprocessing, ModelType: normalizedModelType, }) - failed, finishErr := s.failTask(ctx, task.ID, "insufficient_balance", reserveErr.Error(), task.RunMode == "simulation", reserveErr, parameterPreprocessingMetrics(firstPreprocessing)) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, "insufficient_balance", reserveErr.Error(), task.RunMode == "simulation", reserveErr, parameterPreprocessingMetrics(firstPreprocessing)) if finishErr != nil { return Result{}, finishErr } @@ -406,19 +494,46 @@ candidatesLoop: break candidatesLoop } candidateBody := preprocessing.Body - response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record) + candidatePricing := pricingByCandidate[pricingCandidateKey(candidate)] + response, err := s.runCandidate(ctx, task, user, candidateBody, preprocessing.Log, candidate, candidatePricing, nextAttemptNo, onDelta, responseExecution, singleSourceProtected, runnerPolicy.CacheAffinityPolicy, cacheAffinityKeys.Record) if err == nil { attemptNo = nextAttemptNo var billings []any + finalAmount := fixedAmount(0) + finalAmountText := "" + pricingSnapshot := map[string]any{"pricingVersion": pricingVersionV2, "source": "simulation"} if task.RunMode == "production" { - pricing := pricingByCandidate[pricingCandidateKey(candidate)] + billingMode := normalizedBillingEngineMode(s.cfg.BillingEngineMode) var billingErr error - billings, _, _, billingErr = s.billingsWithResolvedPricingV2(ctx, user, task.Kind, candidateBody, candidate, response, false, pricing) - if billingErr != nil { - // The upstream result may already exist. Preserve the reservation for manual review. - walletReservationFinalized = true - return Result{}, billingErr + if billingMode == "observe" { + billings = s.billings(ctx, user, task.Kind, candidateBody, candidate, response, false) + finalAmount = billingItemsFixedTotal(billings) + pricingSnapshot = map[string]any{ + "pricingVersion": "legacy-observe", "observedPricingVersion": pricingVersionV2, + "observedPricing": candidatePricing.Snapshot, + } + } else { + billings, finalAmount, _, billingErr = s.billingsWithResolvedPricingV2(ctx, user, task.Kind, candidateBody, candidate, response, false, candidatePricing) + pricingSnapshot = candidatePricing.Snapshot } + if billingErr != nil { + review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{ + TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: response.AttemptID, TaskStatus: "succeeded", + Code: "billing_calculation_failed", Message: "billing calculation requires manual review", + Result: response.Result, RequestID: response.RequestID, + PricingSnapshot: candidatePricing.Snapshot, + RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody), + ResponseStartedAt: response.ResponseStartedAt, ResponseFinishedAt: response.ResponseFinishedAt, + ResponseDurationMS: response.ResponseDurationMS, + }) + if reviewErr != nil { + return Result{}, reviewErr + } + walletReservationFinalized = true + s.logger.Warn("task succeeded but billing requires manual review", "taskID", task.ID, "error_category", "billing_calculation_failed") + return Result{Task: review, Output: response.Result}, nil + } + finalAmountText = finalAmount.String() } else { billings = s.billings(ctx, user, task.Kind, candidateBody, candidate, response, true) } @@ -427,38 +542,32 @@ candidatesLoop: record.Metrics = mergeMetrics(record.Metrics, parameterPreprocessingMetrics(preprocessing.Log)) record.Metrics = s.withAttemptHistory(ctx, task.ID, record.Metrics) finished, finishErr := s.store.FinishTaskSuccess(ctx, store.FinishTaskSuccessInput{ - TaskID: task.ID, - Result: response.Result, - Billings: billings, - RequestID: record.RequestID, - ResolvedModel: record.ResolvedModel, - Usage: record.Usage, - Metrics: record.Metrics, - BillingSummary: record.BillingSummary, - FinalChargeAmount: record.FinalChargeAmount, - ResponseStartedAt: record.ResponseStartedAt, - ResponseFinishedAt: record.ResponseFinishedAt, - ResponseDurationMS: record.ResponseDurationMS, + TaskID: task.ID, + ExecutionToken: task.ExecutionToken, + AttemptID: response.AttemptID, + Result: response.Result, + Billings: billings, + RequestID: record.RequestID, + ResolvedModel: record.ResolvedModel, + Usage: record.Usage, + Metrics: record.Metrics, + BillingSummary: record.BillingSummary, + FinalChargeAmount: record.FinalChargeAmount, + FinalChargeAmountText: finalAmountText, + BillingCurrency: stringFromAny(record.BillingSummary["currency"]), + PricingSnapshot: pricingSnapshot, + RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody), + ResponseStartedAt: record.ResponseStartedAt, + ResponseFinishedAt: record.ResponseFinishedAt, + ResponseDurationMS: record.ResponseDurationMS, }) if finishErr != nil { return Result{}, finishErr } - if finished.FinalChargeAmount > 0 { - walletReservationFinalized = true - if settleErr := s.store.SettleTaskBilling(ctx, finished); settleErr != nil { - return Result{}, settleErr - } - } else if len(walletReservations) > 0 { - if releaseErr := s.store.ReleaseTaskBillingReservations(ctx, walletReservations, "task_billing_zero"); releaseErr != nil { - return Result{}, releaseErr - } - walletReservationFinalized = true - } walletReservationFinalized = true - if finished.FinalChargeAmount > 0 { - if err := s.emit(ctx, task.ID, "task.billing.settled", "succeeded", "billing", 0.98, "task billing settled", map[string]any{ - "amount": finished.FinalChargeAmount, - "currency": stringFromAny(record.BillingSummary["currency"]), + if finished.BillingStatus == "pending" { + if err := s.emit(ctx, task.ID, "task.billing.pending", "succeeded", "billing", 0.98, "task billing queued", map[string]any{ + "amount": finished.FinalChargeAmount, "currency": finished.BillingCurrency, }, isSimulation(task, candidate)); err != nil { return Result{}, err } @@ -475,6 +584,21 @@ candidatesLoop: } return Result{Task: finished, Output: response.Result}, nil } + var submissionUnknown *upstreamSubmissionUnknownError + if errors.As(err, &submissionUnknown) { + review, reviewErr := s.store.FinishTaskManualReview(context.WithoutCancel(ctx), store.FinishTaskManualReviewInput{ + TaskID: task.ID, ExecutionToken: task.ExecutionToken, AttemptID: submissionUnknown.AttemptID, TaskStatus: "failed", + Code: "upstream_submission_unknown", Message: submissionUnknown.Error(), + PricingSnapshot: candidatePricing.Snapshot, + RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, candidateBody), + }) + if reviewErr != nil { + return Result{}, reviewErr + } + walletReservationFinalized = true + s.logger.Warn("upstream submission requires manual review", "taskID", task.ID, "attemptID", submissionUnknown.AttemptID, "error_category", "upstream_submission_unknown") + return Result{Task: review, Output: review.Result}, submissionUnknown + } if isLocalRateLimitError(err) { lastErr = err candidateErr = err @@ -622,10 +746,12 @@ candidatesLoop: if lastPreprocessing != nil { extraMetrics = append(extraMetrics, parameterPreprocessingMetrics(*lastPreprocessing)) } - failed, err := s.failTask(ctx, task.ID, code, message, task.RunMode == "simulation", lastErr, extraMetrics...) + failed, err := s.failTask(ctx, task.ID, task.ExecutionToken, code, message, task.RunMode == "simulation", lastErr, extraMetrics...) if err != nil { return Result{}, err } + // FinishTaskFailure atomically transfers ownership of any reservation to the release Outbox. + walletReservationFinalized = true return Result{Task: failed, Output: failed.Result}, lastErr } @@ -633,7 +759,46 @@ func pricingCandidateKey(candidate store.RuntimeModelCandidate) string { return firstNonEmptyString(candidate.PlatformModelID, candidate.PlatformID+":"+candidate.ModelName) } -func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) { +func normalizedBillingEngineMode(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "enforce", "hold": + return strings.ToLower(strings.TrimSpace(value)) + default: + return "observe" + } +} + +func (s *Service) maximumLegacyCandidateEstimate(ctx context.Context, user *auth.User, kind string, body map[string]any, candidates []store.RuntimeModelCandidate) ([]any, fixedAmount) { + var maximumItems []any + maximumAmount := fixedAmount(0) + for index, candidate := range candidates { + candidateBody := preprocessRequest(kind, cloneMap(body), candidate) + items := s.estimatedBillings(ctx, user, kind, candidateBody, candidate) + amount := billingItemsFixedTotal(items) + if index == 0 || amount > maximumAmount { + maximumItems = items + maximumAmount = amount + } + } + return maximumItems, maximumAmount +} + +func billingItemsFixedTotal(items []any) fixedAmount { + total := fixedAmount(0) + for _, raw := range items { + line, _ := raw.(map[string]any) + if line == nil { + continue + } + amount, err := fixedAmountFromAny(line["amount"]) + if err == nil && amount > 0 { + total = total.Add(amount) + } + } + return total +} + +func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user *auth.User, body map[string]any, preprocessing parameterPreprocessingLog, candidate store.RuntimeModelCandidate, pricing resolvedPricing, attemptNo int, onDelta clients.StreamDelta, responseExecution responseExecutionContext, singleSourceProtected bool, cacheAffinityPolicy map[string]any, cacheAffinityRecordKeys []string) (clients.Response, error) { simulated := isSimulation(task, candidate) baseAttemptMetrics := mergeMetrics(attemptMetrics(candidate, attemptNo, simulated), parameterPreprocessingMetrics(preprocessing)) reservations := s.rateLimitReservations(ctx, user, candidate, body) @@ -652,16 +817,18 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user defer s.store.ReleaseConcurrencyLeases(context.WithoutCancel(ctx), limitResult.LeaseIDs) attemptID, err := s.store.CreateTaskAttempt(ctx, store.CreateTaskAttemptInput{ - TaskID: task.ID, - AttemptNo: attemptNo, - PlatformID: candidate.PlatformID, - PlatformModelID: candidate.PlatformModelID, - ClientID: candidate.ClientID, - QueueKey: candidate.QueueKey, - Status: "running", - Simulated: simulated, - RequestSnapshot: s.slimTaskRequestSnapshot(task, body), - Metrics: baseAttemptMetrics, + TaskID: task.ID, + AttemptNo: attemptNo, + PlatformID: candidate.PlatformID, + PlatformModelID: candidate.PlatformModelID, + ClientID: candidate.ClientID, + QueueKey: candidate.QueueKey, + Status: "running", + Simulated: simulated, + RequestSnapshot: s.slimTaskRequestSnapshot(task, body), + Metrics: baseAttemptMetrics, + PricingSnapshot: pricing.Snapshot, + RequestFingerprint: pricingRequestFingerprint(task.Kind, task.Model, body), }) if err != nil { return clients.Response{}, fmt.Errorf("create task attempt: %w", err) @@ -731,6 +898,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user publicResponseID = responseExecution.PublicResponseID publicPreviousResponseID = responseExecution.PublicPreviousResponseID } + if err := s.store.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil { + return clients.Response{}, fmt.Errorf("mark upstream submission: %w", err) + } response, err := client.Run(ctx, clients.Request{ Kind: task.Kind, ModelType: candidate.ModelType, @@ -744,7 +914,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user if strings.TrimSpace(remoteTaskID) == "" { return nil } - return s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, attemptID, remoteTaskID, payload) + return s.store.SetTaskRemoteTask(context.WithoutCancel(ctx), task.ID, task.ExecutionToken, attemptID, remoteTaskID, payload) }, Stream: boolFromMap(providerBody, "stream"), StreamDelta: onDelta, @@ -768,6 +938,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user } } if err != nil { + if clients.ErrorResponseMetadata(err).StatusCode > 0 { + _ = s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received") + } retryable := clients.IsRetryable(err) requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated) if responseStartedAt.IsZero() { @@ -796,6 +969,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user ErrorMessage: err.Error(), }) _ = s.emit(ctx, task.ID, "task.attempt.failed", "running", "attempt_failed", 0.45, err.Error(), map[string]any{"attempt": attemptNo, "retryable": retryable, "requestId": requestID, "statusCode": clients.ErrorResponseMetadata(err).StatusCode, "metrics": metrics}, simulated) + if !simulated && clients.ErrorResponseMetadata(err).StatusCode == 0 { + return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: err} + } s.applyCandidateFailurePolicies(ctx, task.ID, candidate, err, simulated, singleSourceProtected) return clients.Response{}, err } @@ -877,19 +1053,7 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user return clients.Response{}, fmt.Errorf("commit rate limit reservations: %w", err) } rateReservationsFinalized = true - if err := s.store.FinishTaskAttempt(ctx, store.FinishTaskAttemptInput{ - AttemptID: attemptID, - Status: "succeeded", - RequestID: response.RequestID, - Usage: usageToMap(response.Usage), - Metrics: mergeMetrics(taskMetrics(task, user, body, candidate, response, simulated), parameterPreprocessingMetrics(preprocessing)), - ResponseSnapshot: response.Result, - ResponseStartedAt: response.ResponseStartedAt, - ResponseFinishedAt: response.ResponseFinishedAt, - ResponseDurationMS: response.ResponseDurationMS, - }); err != nil { - return clients.Response{}, fmt.Errorf("finish task attempt: %w", err) - } + response.AttemptID = attemptID if err := s.store.RecordCacheAffinityObservation(context.WithoutCancel(ctx), store.CacheAffinityObservationInput{ CacheAffinityKey: candidate.CacheAffinity.Key, CacheAffinityKeys: cacheAffinityRecordKeys, @@ -957,7 +1121,7 @@ func (s *Service) clientFor(candidate store.RuntimeModelCandidate, simulated boo return s.clients["openai"] } -func (s *Service) failTask(ctx context.Context, taskID string, code string, message string, simulated bool, cause error, extraMetrics ...map[string]any) (store.GatewayTask, error) { +func (s *Service) failTask(ctx context.Context, taskID string, executionToken string, code string, message string, simulated bool, cause error, extraMetrics ...map[string]any) (store.GatewayTask, error) { requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(cause, simulated) if len(extraMetrics) > 0 { values := append([]map[string]any{metrics}, extraMetrics...) @@ -966,6 +1130,7 @@ func (s *Service) failTask(ctx context.Context, taskID string, code string, mess metrics = s.withAttemptHistory(ctx, taskID, metrics) failed, err := s.store.FinishTaskFailure(ctx, store.FinishTaskFailureInput{ TaskID: taskID, + ExecutionToken: executionToken, Code: code, Message: message, Result: buildFailureResult(code, message, requestID, cause), @@ -1094,7 +1259,7 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway if delay <= 0 { delay = 5 * time.Second } - queued, err := s.store.RequeueTask(ctx, task.ID, delay, candidate.QueueKey) + queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, delay, candidate.QueueKey) if err != nil { return store.GatewayTask{}, 0, err } @@ -1110,7 +1275,7 @@ func (s *Service) requeueRateLimitedTask(ctx context.Context, task store.Gateway } func (s *Service) requeueInterruptedAsyncTask(ctx context.Context, task store.GatewayTask) (store.GatewayTask, error) { - queued, err := s.store.RequeueTask(ctx, task.ID, 0, "") + queued, err := s.store.RequeueTask(ctx, task.ID, task.ExecutionToken, 0, "") if err != nil { return store.GatewayTask{}, err } diff --git a/apps/api/internal/runner/wallet_execute_test.go b/apps/api/internal/runner/wallet_execute_test.go index d4d9460..9ec1493 100644 --- a/apps/api/internal/runner/wallet_execute_test.go +++ b/apps/api/internal/runner/wallet_execute_test.go @@ -182,6 +182,20 @@ func TestExecuteWithMockClientRejectsConcurrentTasksBeyondWalletBalance(t *testi if got := mockClient.calls.Load(); got != 1 { t.Fatalf("mock client calls = %d, want 1", got) } + settlements, err := db.ClaimBillingSettlements(ctx, "wallet-execute-test", store.BillingSettlementBatchSize, store.BillingSettlementLockTimeout) + if err != nil { + t.Fatalf("claim billing settlements: %v", err) + } + processed := 0 + for _, settlement := range settlements { + if err := db.ProcessBillingSettlement(ctx, settlement); err != nil { + t.Fatalf("process billing settlement %s: %v", settlement.ID, err) + } + processed++ + } + if processed != 1 { + t.Fatalf("processed billing settlements = %d, want 1", processed) + } summary, err := db.GetWalletSummary(ctx, user, "resource") if err != nil { diff --git a/apps/api/internal/securityevents/metrics.go b/apps/api/internal/securityevents/metrics.go index b7a8e88..6bc24b3 100644 --- a/apps/api/internal/securityevents/metrics.go +++ b/apps/api/internal/securityevents/metrics.go @@ -18,25 +18,32 @@ type MetricsSnapshotProvider interface { type DynamicMetricsSnapshotProvider interface { MetricsSnapshotProvider SecurityEventConnection(context.Context) (store.SecurityEventConnection, error) + BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error) } type Metrics struct { - accepted atomic.Uint64 - rejected atomic.Uint64 - duplicate atomic.Uint64 - sessionsDeleted atomic.Uint64 - watermarkRejected atomic.Uint64 - verificationAccepted atomic.Uint64 - heartbeatAccepted atomic.Uint64 - heartbeatFailed atomic.Uint64 - introspectionActive atomic.Uint64 - introspectionInactive atomic.Uint64 - introspectionFailed atomic.Uint64 - jwksSSFFailed atomic.Uint64 - jwksOIDCFailed atomic.Uint64 - processingCount atomic.Uint64 - processingNanos atomic.Uint64 - processingBuckets [6]atomic.Uint64 + accepted atomic.Uint64 + rejected atomic.Uint64 + duplicate atomic.Uint64 + sessionsDeleted atomic.Uint64 + watermarkRejected atomic.Uint64 + verificationAccepted atomic.Uint64 + heartbeatAccepted atomic.Uint64 + heartbeatFailed atomic.Uint64 + introspectionActive atomic.Uint64 + introspectionInactive atomic.Uint64 + introspectionFailed atomic.Uint64 + jwksSSFFailed atomic.Uint64 + jwksOIDCFailed atomic.Uint64 + processingCount atomic.Uint64 + processingNanos atomic.Uint64 + processingBuckets [6]atomic.Uint64 + billingSettlementCompleted atomic.Uint64 + billingSettlementRetry atomic.Uint64 + billingManualReview atomic.Uint64 + billingEstimateFailed atomic.Uint64 + billingIdempotentReplay atomic.Uint64 + billingPricingUnavailable atomic.Uint64 } var processingDurationBounds = [...]time.Duration{ @@ -99,6 +106,23 @@ func (m *Metrics) ObserveJWKSRefreshFailure(source string) { m.jwksOIDCFailed.Add(1) } +func (m *Metrics) ObserveBillingEvent(event string) { + switch event { + case "settlement_completed": + m.billingSettlementCompleted.Add(1) + case "settlement_retry": + m.billingSettlementRetry.Add(1) + case "manual_review": + m.billingManualReview.Add(1) + case "estimate_failed": + m.billingEstimateFailed.Add(1) + case "idempotent_replay": + m.billingIdempotentReplay.Add(1) + case "pricing_unavailable": + m.billingPricingUnavailable.Add(1) + } +} + func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience string, enabled bool) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mode := "disabled" @@ -113,6 +137,17 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str return } } + billing := store.BillingMetricsSnapshot{} + if billingProvider, ok := provider.(interface { + BillingMetrics(context.Context) (store.BillingMetricsSnapshot, error) + }); ok { + var err error + billing, err = billingProvider.BillingMetrics(r.Context()) + if err != nil { + http.Error(w, "metrics unavailable", http.StatusServiceUnavailable) + return + } + } w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") outcomeCounters(w, "easyai_gateway_ssf_receipts_total", "Received SETs by bounded outcome.", []outcomeValue{ {"accepted", m.accepted.Load()}, {"rejected", m.rejected.Load()}, {"duplicate", m.duplicate.Load()}, @@ -156,6 +191,27 @@ func (m *Metrics) Handler(provider MetricsSnapshotProvider, issuer, audience str } fmt.Fprintf(w, "easyai_gateway_ssf_mode{mode=\"%s\"} %d\n", modeName, value) } + fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_backlog Current unsettled Outbox records.") + fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_backlog gauge") + fmt.Fprintf(w, "easyai_gateway_billing_settlement_backlog %d\n", billing.SettlementBacklog) + fmt.Fprintln(w, "# HELP easyai_gateway_billing_settlement_delay_seconds Age of the oldest unsettled Outbox record.") + fmt.Fprintln(w, "# TYPE easyai_gateway_billing_settlement_delay_seconds gauge") + fmt.Fprintf(w, "easyai_gateway_billing_settlement_delay_seconds %.6f\n", billing.SettlementDelaySecs) + fmt.Fprintln(w, "# HELP easyai_gateway_billing_manual_review Current billing records requiring manual review.") + fmt.Fprintln(w, "# TYPE easyai_gateway_billing_manual_review gauge") + fmt.Fprintf(w, "easyai_gateway_billing_manual_review %d\n", billing.ManualReview) + fmt.Fprintln(w, "# HELP easyai_gateway_billing_orphan_frozen Wallets whose frozen amount differs from active reservations.") + fmt.Fprintln(w, "# TYPE easyai_gateway_billing_orphan_frozen gauge") + fmt.Fprintf(w, "easyai_gateway_billing_orphan_frozen %d\n", billing.OrphanFrozen) + fmt.Fprintln(w, "# HELP easyai_gateway_billing_pricing_unavailable_tasks Current tasks rejected because no effective price was available.") + fmt.Fprintln(w, "# TYPE easyai_gateway_billing_pricing_unavailable_tasks gauge") + fmt.Fprintf(w, "easyai_gateway_billing_pricing_unavailable_tasks %d\n", billing.PricingUnavailable) + plainCounter(w, "easyai_gateway_billing_settlements_completed_total", "Billing settlements completed by this process.", m.billingSettlementCompleted.Load()) + plainCounter(w, "easyai_gateway_billing_settlement_retries_total", "Billing settlement retries scheduled by this process.", m.billingSettlementRetry.Load()) + plainCounter(w, "easyai_gateway_billing_manual_review_transitions_total", "Billing records transitioned to manual review by this process.", m.billingManualReview.Load()) + plainCounter(w, "easyai_gateway_billing_estimate_failures_total", "Pricing estimate requests that failed.", m.billingEstimateFailed.Load()) + plainCounter(w, "easyai_gateway_billing_idempotent_replays_total", "Generation requests replayed idempotently.", m.billingIdempotentReplay.Load()) + plainCounter(w, "easyai_gateway_billing_pricing_unavailable_total", "Pricing requests rejected because no effective price was available.", m.billingPricingUnavailable.Load()) }) } diff --git a/apps/api/internal/store/billing_settlements.go b/apps/api/internal/store/billing_settlements.go new file mode 100644 index 0000000..338e6bf --- /dev/null +++ b/apps/api/internal/store/billing_settlements.go @@ -0,0 +1,637 @@ +package store + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" +) + +const ( + BillingSettlementBatchSize = 50 + BillingSettlementLockTimeout = 120 * time.Second + BillingSettlementMaxAttempts = 20 +) + +var ErrBillingSettlementNotRetryable = errors.New("billing settlement is not retryable") + +type BillingSettlement struct { + ID string `json:"id"` + TaskID string `json:"taskId"` + Action string `json:"action"` + Amount float64 `json:"amount"` + Currency string `json:"currency"` + Status string `json:"status"` + Attempts int `json:"attempts"` + NextAttemptAt time.Time `json:"nextAttemptAt"` + LockedBy string `json:"lockedBy,omitempty"` + LockedAt string `json:"lockedAt,omitempty"` + LastErrorCode string `json:"lastErrorCode,omitempty"` + LastErrorMessage string `json:"lastErrorMessage,omitempty"` + ManualReviewReason string `json:"manualReviewReason,omitempty"` + PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` + Payload map[string]any `json:"payload,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + LockToken string `json:"-"` + amountExact string +} + +type BillingSettlementListFilter struct { + Status string + Action string + Page int + PageSize int +} + +type BillingSettlementListResult struct { + Items []BillingSettlement `json:"items"` + Total int `json:"total"` + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +type BillingMetricsSnapshot struct { + SettlementBacklog int64 + SettlementDelaySecs float64 + ManualReview int64 + OrphanFrozen int64 + PricingUnavailable int64 +} + +func (s *Store) BillingMetrics(ctx context.Context) (BillingMetricsSnapshot, error) { + var snapshot BillingMetricsSnapshot + err := s.pool.QueryRow(ctx, ` +WITH open_outbox AS ( + SELECT created_at + FROM settlement_outbox + WHERE status IN ('pending', 'processing', 'retryable_failed') +), active_reservations AS ( + SELECT reserve.account_id, COALESCE(SUM(reserve.amount), 0) AS amount + FROM gateway_wallet_transactions reserve + WHERE reserve.transaction_type = 'reserve' + AND NOT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions release + WHERE release.account_id = reserve.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = reserve.idempotency_key || ':release' + ) + GROUP BY reserve.account_id +) +SELECT + (SELECT COUNT(*) FROM open_outbox), + COALESCE((SELECT EXTRACT(EPOCH FROM now() - MIN(created_at)) FROM open_outbox), 0), + (SELECT COUNT(*) FROM gateway_tasks WHERE billing_status = 'manual_review'), + (SELECT COUNT(*) FROM gateway_wallet_accounts account + LEFT JOIN active_reservations reservation ON reservation.account_id = account.id + WHERE account.frozen_balance <> COALESCE(reservation.amount, 0)), + (SELECT COUNT(*) FROM gateway_tasks WHERE error_code = 'pricing_unavailable')`).Scan( + &snapshot.SettlementBacklog, + &snapshot.SettlementDelaySecs, + &snapshot.ManualReview, + &snapshot.OrphanFrozen, + &snapshot.PricingUnavailable, + ) + return snapshot, err +} + +func (s *Store) ClaimBillingSettlements(ctx context.Context, workerID string, limit int, staleAfter time.Duration) ([]BillingSettlement, error) { + if limit <= 0 || limit > BillingSettlementBatchSize { + limit = BillingSettlementBatchSize + } + if staleAfter <= 0 { + staleAfter = BillingSettlementLockTimeout + } + rows, err := s.pool.Query(ctx, ` +WITH picked AS ( + SELECT id + FROM settlement_outbox + WHERE ( + status IN ('pending', 'retryable_failed') + AND next_attempt_at <= now() + ) + OR ( + status = 'processing' + AND locked_at < now() - ($3::int * interval '1 second') + ) + ORDER BY next_attempt_at ASC, created_at ASC + LIMIT $2 + FOR UPDATE SKIP LOCKED +) +UPDATE settlement_outbox outbox +SET status = 'processing', + attempts = outbox.attempts + 1, + locked_by = $1, + lock_token = gen_random_uuid(), + locked_at = now(), + updated_at = now() +FROM picked +WHERE outbox.id = picked.id +RETURNING outbox.id::text, outbox.task_id::text, outbox.action, outbox.amount::text, + outbox.currency, outbox.status, outbox.attempts, outbox.next_attempt_at, + COALESCE(outbox.locked_by, ''), COALESCE(outbox.lock_token::text, ''), + COALESCE(outbox.locked_at::text, ''), COALESCE(outbox.last_error_code, ''), + COALESCE(outbox.last_error_message, ''), COALESCE(outbox.manual_review_reason, ''), + outbox.pricing_snapshot, outbox.payload, COALESCE(outbox.completed_at::text, ''), + outbox.created_at, outbox.updated_at`, + workerID, limit, int(staleAfter/time.Second)) + if err != nil { + return nil, err + } + defer rows.Close() + items := make([]BillingSettlement, 0) + for rows.Next() { + item, err := scanBillingSettlement(rows) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, rows.Err() +} + +func (s *Store) ProcessBillingSettlement(ctx context.Context, settlement BillingSettlement) error { + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var action string + var amount string + var currency string + var taskID string + var taskStatus string + var gatewayUserID string + var gatewayTenantID string + var userID string + var tenantID string + var tenantKey string + err := tx.QueryRow(ctx, ` +SELECT outbox.action, outbox.amount::text, outbox.currency, outbox.task_id::text, + task.status, COALESCE(task.gateway_user_id::text, ''), + COALESCE(task.gateway_tenant_id::text, ''), task.user_id, + COALESCE(task.tenant_id, ''), COALESCE(task.tenant_key, '') +FROM settlement_outbox outbox +JOIN gateway_tasks task ON task.id = outbox.task_id +WHERE outbox.id = $1::uuid + AND outbox.status = 'processing' + AND outbox.lock_token = $2::uuid +FOR UPDATE OF outbox, task`, settlement.ID, settlement.LockToken).Scan( + &action, &amount, ¤cy, &taskID, &taskStatus, &gatewayUserID, + &gatewayTenantID, &userID, &tenantID, &tenantKey, + ) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET billing_status = 'processing', billing_updated_at = now(), updated_at = now() +WHERE id = $1::uuid`, taskID); err != nil { + return err + } + switch action { + case "settle": + if taskStatus != "succeeded" { + return fmt.Errorf("settle action requires succeeded task") + } + if err := settleBillingOutboxTx(ctx, tx, taskID, gatewayUserID, gatewayTenantID, userID, tenantID, tenantKey, currency, amount); err != nil { + return err + } + return completeBillingOutboxTx(ctx, tx, settlement.ID, settlement.LockToken, taskID, "settled") + case "release": + if err := releaseBillingOutboxTx(ctx, tx, taskID, gatewayUserID, gatewayTenantID, currency); err != nil { + return err + } + return completeBillingOutboxTx(ctx, tx, settlement.ID, settlement.LockToken, taskID, "released") + default: + return fmt.Errorf("unsupported billing settlement action %q", action) + } + }) +} + +func settleBillingOutboxTx( + ctx context.Context, + tx pgx.Tx, + taskID string, + gatewayUserID string, + gatewayTenantID string, + userID string, + tenantID string, + tenantKey string, + currency string, + amount string, +) error { + if gatewayUserID == "" { + return fmt.Errorf("task %s has no gateway wallet user", taskID) + } + if _, err := tx.Exec(ctx, ` +INSERT INTO gateway_wallet_accounts ( + gateway_tenant_id, gateway_user_id, tenant_id, tenant_key, user_id, currency +) +VALUES (NULLIF($1, '')::uuid, $2::uuid, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), $6) +ON CONFLICT (gateway_user_id, currency) DO NOTHING`, + gatewayTenantID, gatewayUserID, tenantID, tenantKey, userID, currency); err != nil { + return err + } + if err := ensureWalletAccountAuditGuard(ctx, tx, gatewayUserID, currency); err != nil { + return err + } + var accountID string + var balanceBefore string + var frozenBefore string + if err := tx.QueryRow(ctx, ` +SELECT id::text, balance::text, frozen_balance::text +FROM gateway_wallet_accounts +WHERE gateway_user_id = $1::uuid AND currency = $2 +FOR UPDATE`, gatewayUserID, currency).Scan(&accountID, &balanceBefore, &frozenBefore); err != nil { + return err + } + var alreadySettled bool + if err := tx.QueryRow(ctx, ` +SELECT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions + WHERE account_id = $1::uuid AND idempotency_key = $2 +)`, accountID, billingIdempotencyKey(taskID)).Scan(&alreadySettled); err != nil { + return err + } + if alreadySettled { + return nil + } + + var reservationKey string + var reservedAmount string + if err := tx.QueryRow(ctx, ` +SELECT COALESCE(reserve.idempotency_key, ''), reserve.amount::text +FROM gateway_wallet_transactions reserve +WHERE reserve.account_id = $1::uuid + AND reserve.reference_type = 'gateway_task' + AND reserve.reference_id = $2 + AND reserve.transaction_type = 'reserve' + AND NOT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions release + WHERE release.account_id = reserve.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = reserve.idempotency_key || ':release' + ) +ORDER BY reserve.created_at DESC +LIMIT 1`, accountID, taskID).Scan(&reservationKey, &reservedAmount); errors.Is(err, pgx.ErrNoRows) { + reservationKey = "" + reservedAmount = "0" + } else if err != nil { + return err + } + + var balanceAfter string + var frozenAfter string + err := tx.QueryRow(ctx, ` +UPDATE gateway_wallet_accounts +SET balance = balance - $2::numeric, + total_spent = total_spent + $2::numeric, + frozen_balance = GREATEST(0, frozen_balance - $3::numeric), + updated_at = now() +WHERE id = $1::uuid + AND balance - frozen_balance + $3::numeric >= $2::numeric +RETURNING balance::text, frozen_balance::text`, + accountID, amount, reservedAmount).Scan(&balanceAfter, &frozenAfter) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%w: task %s settlement amount exceeds spendable balance", ErrInsufficientWalletBalance, taskID) + } + if err != nil { + return err + } + + var reservedPositive bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, reservedAmount).Scan(&reservedPositive); err != nil { + return err + } + if reservedPositive { + releaseMetadata, _ := json.Marshal(map[string]any{ + "taskId": taskID, "reason": "task_billing_settled", + "reserved": reservedAmount, "frozenBefore": frozenBefore, "frozenAfter": frozenAfter, + }) + if _, err := tx.Exec(ctx, ` +INSERT INTO gateway_wallet_transactions ( + account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type, + amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata +) +VALUES ( + $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release', + $4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb +) +ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`, + accountID, gatewayTenantID, gatewayUserID, reservedAmount, balanceBefore, + billingReservationReleaseIdempotencyKey(reservationKey), taskID, string(releaseMetadata)); err != nil { + return err + } + } + metadata, _ := json.Marshal(map[string]any{ + "taskId": taskID, "reservedAmount": reservedAmount, + "frozenBefore": frozenBefore, "frozenAfter": frozenAfter, + }) + _, err = tx.Exec(ctx, ` +INSERT INTO gateway_wallet_transactions ( + account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type, + amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata +) +VALUES ( + $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'debit', 'task_billing', + $4::numeric, $5::numeric, $6::numeric, $7, 'gateway_task', $8, $9::jsonb +) +ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`, + accountID, gatewayTenantID, gatewayUserID, amount, balanceBefore, balanceAfter, + billingIdempotencyKey(taskID), taskID, string(metadata)) + return err +} + +func releaseBillingOutboxTx(ctx context.Context, tx pgx.Tx, taskID string, gatewayUserID string, gatewayTenantID string, currency string) error { + if gatewayUserID == "" { + return nil + } + var accountID string + var balance string + var frozenBefore string + err := tx.QueryRow(ctx, ` +SELECT id::text, balance::text, frozen_balance::text +FROM gateway_wallet_accounts +WHERE gateway_user_id = $1::uuid AND currency = $2 +FOR UPDATE`, gatewayUserID, currency).Scan(&accountID, &balance, &frozenBefore) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + var reservationKey string + var reservedAmount string + err = tx.QueryRow(ctx, ` +SELECT COALESCE(reserve.idempotency_key, ''), reserve.amount::text +FROM gateway_wallet_transactions reserve +WHERE reserve.account_id = $1::uuid + AND reserve.reference_type = 'gateway_task' + AND reserve.reference_id = $2 + AND reserve.transaction_type = 'reserve' + AND NOT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions release + WHERE release.account_id = reserve.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = reserve.idempotency_key || ':release' + ) +ORDER BY reserve.created_at DESC +LIMIT 1`, accountID, taskID).Scan(&reservationKey, &reservedAmount) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return err + } + var frozenAfter string + if err := tx.QueryRow(ctx, ` +UPDATE gateway_wallet_accounts +SET frozen_balance = frozen_balance - $2::numeric, + updated_at = now() +WHERE id = $1::uuid + AND frozen_balance >= $2::numeric +RETURNING frozen_balance::text`, accountID, reservedAmount).Scan(&frozenAfter); err != nil { + return err + } + metadata, _ := json.Marshal(map[string]any{ + "taskId": taskID, "reason": "task_terminal_release", + "reserved": reservedAmount, "frozenBefore": frozenBefore, "frozenAfter": frozenAfter, + }) + _, err = tx.Exec(ctx, ` +INSERT INTO gateway_wallet_transactions ( + account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type, + amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata +) +VALUES ( + $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release', + $4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb +) +ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`, + accountID, gatewayTenantID, gatewayUserID, reservedAmount, balance, + billingReservationReleaseIdempotencyKey(reservationKey), taskID, string(metadata)) + return err +} + +func completeBillingOutboxTx(ctx context.Context, tx pgx.Tx, settlementID string, lockToken string, taskID string, billingStatus string) error { + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET billing_status = $2, + reservation_amount = 0, + billing_updated_at = now(), + billing_settled_at = CASE WHEN $2 = 'settled' THEN now() ELSE billing_settled_at END, + updated_at = now() +WHERE id = $1::uuid`, taskID, billingStatus); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` +INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated) +SELECT task.id, + COALESCE((SELECT MAX(event.seq) + 1 FROM gateway_task_events event WHERE event.task_id = task.id), 1), + CASE WHEN $2 = 'settled' THEN 'task.billing.settled' ELSE 'task.billing.released' END, + task.status, + 'billing', + 1, + CASE WHEN $2 = 'settled' THEN 'task billing settled' ELSE 'task billing reservation released' END, + jsonb_build_object('settlementId', $3::text, 'billingStatus', $2::text), + task.run_mode = 'simulation' +FROM gateway_tasks task +WHERE task.id = $1::uuid`, taskID, billingStatus, settlementID); err != nil { + return err + } + tag, err := tx.Exec(ctx, ` +UPDATE settlement_outbox +SET status = 'completed', + completed_at = now(), + locked_by = NULL, + lock_token = NULL, + locked_at = NULL, + last_error_code = NULL, + last_error_message = NULL, + updated_at = now() +WHERE id = $1::uuid + AND status = 'processing' + AND lock_token = $2::uuid`, settlementID, lockToken) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return pgx.ErrNoRows + } + return nil +} + +func (s *Store) MarkBillingSettlementFailed(ctx context.Context, settlement BillingSettlement, code string, message string, delay time.Duration) error { + if delay < time.Second { + delay = time.Second + } + if delay > 15*time.Minute { + delay = 15 * time.Minute + } + manualReview := settlement.Attempts >= BillingSettlementMaxAttempts + status := "retryable_failed" + taskStatus := "retryable_failed" + manualReason := "" + if manualReview { + status = "manual_review" + taskStatus = "manual_review" + manualReason = "maximum settlement attempts exceeded" + } + return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + tag, err := tx.Exec(ctx, ` +UPDATE settlement_outbox +SET status = $3, + next_attempt_at = now() + ($4::int * interval '1 second'), + locked_by = NULL, + lock_token = NULL, + locked_at = NULL, + last_error_code = NULLIF($5, ''), + last_error_message = NULLIF($6, ''), + manual_review_reason = NULLIF($7, ''), + updated_at = now() +WHERE id = $1::uuid + AND lock_token = $2::uuid + AND status = 'processing'`, + settlement.ID, settlement.LockToken, status, int(delay/time.Second), code, message, manualReason) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return pgx.ErrNoRows + } + _, err = tx.Exec(ctx, ` +UPDATE gateway_tasks +SET billing_status = $2, + billing_updated_at = now(), + updated_at = now() +WHERE id = $1::uuid`, settlement.TaskID, taskStatus) + return err + }) +} + +func (s *Store) ListBillingSettlements(ctx context.Context, filter BillingSettlementListFilter) (BillingSettlementListResult, error) { + page := filter.Page + if page <= 0 { + page = 1 + } + pageSize := filter.PageSize + if pageSize <= 0 { + pageSize = 50 + } + if pageSize > 100 { + pageSize = 100 + } + status := strings.TrimSpace(filter.Status) + action := strings.TrimSpace(filter.Action) + var total int + if err := s.pool.QueryRow(ctx, ` +SELECT count(*) +FROM settlement_outbox +WHERE (NULLIF($1, '') IS NULL OR status = $1) + AND (NULLIF($2, '') IS NULL OR action = $2)`, status, action).Scan(&total); err != nil { + return BillingSettlementListResult{}, err + } + rows, err := s.pool.Query(ctx, ` +SELECT id::text, task_id::text, action, amount::text, currency, status, attempts, + next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''), + COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''), + COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''), + pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at +FROM settlement_outbox +WHERE (NULLIF($1, '') IS NULL OR status = $1) + AND (NULLIF($2, '') IS NULL OR action = $2) +ORDER BY created_at DESC +LIMIT $3 OFFSET $4`, status, action, pageSize, (page-1)*pageSize) + if err != nil { + return BillingSettlementListResult{}, err + } + defer rows.Close() + items := make([]BillingSettlement, 0) + for rows.Next() { + item, err := scanBillingSettlement(rows) + if err != nil { + return BillingSettlementListResult{}, err + } + items = append(items, item) + } + if err := rows.Err(); err != nil { + return BillingSettlementListResult{}, err + } + return BillingSettlementListResult{Items: items, Total: total, Page: page, PageSize: pageSize}, nil +} + +func (s *Store) RetryBillingSettlementTx(ctx context.Context, tx Tx, id string, idempotencyKeyHash string) (BillingSettlement, bool, error) { + current, err := scanBillingSettlement(tx.QueryRow(ctx, ` +SELECT id::text, task_id::text, action, amount::text, currency, status, attempts, + next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''), + COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''), + COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''), + pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at +FROM settlement_outbox +WHERE id = $1::uuid +FOR UPDATE`, id)) + if err != nil { + return BillingSettlement{}, false, err + } + var recordedHash string + if err := tx.QueryRow(ctx, ` +SELECT COALESCE(retry_idempotency_key_hash, '') +FROM settlement_outbox +WHERE id = $1::uuid`, id).Scan(&recordedHash); err != nil { + return BillingSettlement{}, false, err + } + if recordedHash != "" && recordedHash == idempotencyKeyHash { + return current, true, nil + } + if current.Status != "retryable_failed" && current.Status != "manual_review" { + return BillingSettlement{}, false, ErrBillingSettlementNotRetryable + } + item, err := scanBillingSettlement(tx.QueryRow(ctx, ` +UPDATE settlement_outbox +SET status = 'pending', + next_attempt_at = now(), + locked_by = NULL, + lock_token = NULL, + locked_at = NULL, + manual_review_reason = NULL, + retry_idempotency_key_hash = $2, + retry_requested_at = now(), + updated_at = now() +WHERE id = $1::uuid + AND status IN ('retryable_failed', 'manual_review') +RETURNING id::text, task_id::text, action, amount::text, currency, status, attempts, + next_attempt_at, COALESCE(locked_by, ''), COALESCE(lock_token::text, ''), + COALESCE(locked_at::text, ''), COALESCE(last_error_code, ''), + COALESCE(last_error_message, ''), COALESCE(manual_review_reason, ''), + pricing_snapshot, payload, COALESCE(completed_at::text, ''), created_at, updated_at`, id, idempotencyKeyHash)) + if err != nil { + return BillingSettlement{}, false, err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET billing_status = 'pending', billing_updated_at = now(), updated_at = now() +WHERE id = $1::uuid`, item.TaskID); err != nil { + return BillingSettlement{}, false, err + } + return item, false, nil +} + +func scanBillingSettlement(scanner taskScanner) (BillingSettlement, error) { + var item BillingSettlement + var pricingSnapshotBytes []byte + var payloadBytes []byte + if err := scanner.Scan( + &item.ID, &item.TaskID, &item.Action, &item.amountExact, &item.Currency, + &item.Status, &item.Attempts, &item.NextAttemptAt, &item.LockedBy, &item.LockToken, + &item.LockedAt, &item.LastErrorCode, &item.LastErrorMessage, &item.ManualReviewReason, + &pricingSnapshotBytes, &payloadBytes, &item.CompletedAt, &item.CreatedAt, &item.UpdatedAt, + ); err != nil { + return BillingSettlement{}, err + } + item.Amount, _ = strconv.ParseFloat(item.amountExact, 64) + item.PricingSnapshot = decodeObject(pricingSnapshotBytes) + item.Payload = decodeObject(payloadBytes) + return item, nil +} diff --git a/apps/api/internal/store/billing_v2_integration_test.go b/apps/api/internal/store/billing_v2_integration_test.go new file mode 100644 index 0000000..e078029 --- /dev/null +++ b/apps/api/internal/store/billing_v2_integration_test.go @@ -0,0 +1,265 @@ +package store + +import ( + "context" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/auth" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func TestTaskIdempotencyAndExecutionLease(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-v2-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID} + input := CreateTaskInput{ + Kind: "images.generations", Model: "billing-v2-model", RunMode: "simulation", + Request: map[string]any{"model": "billing-v2-model", "prompt": "lease"}, + IdempotencyKeyHash: "key-hash-" + uuid.NewString(), IdempotencyRequestHash: "request-a", + } + created, err := db.CreateTaskIdempotent(ctx, input, user) + if err != nil || created.Replayed { + t.Fatalf("create task result=%+v err=%v", created, err) + } + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.Task.ID) + }) + + replayed, err := db.CreateTaskIdempotent(ctx, input, user) + if err != nil || !replayed.Replayed || replayed.Task.ID != created.Task.ID { + t.Fatalf("replay result=%+v err=%v", replayed, err) + } + input.IdempotencyRequestHash = "request-b" + if _, err := db.CreateTaskIdempotent(ctx, input, user); !errors.Is(err, ErrIdempotencyKeyReused) { + t.Fatalf("different request error=%v", err) + } + + firstToken := uuid.NewString() + claimed, err := db.ClaimTaskExecution(ctx, created.Task.ID, firstToken, 5*time.Minute) + if err != nil || claimed.ExecutionToken != firstToken { + t.Fatalf("first claim task=%+v err=%v", claimed, err) + } + if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, firstToken, 5*time.Minute); err != nil { + t.Fatalf("renew first lease: %v", err) + } + if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.Task.ID); err != nil { + t.Fatal(err) + } + secondToken := uuid.NewString() + if _, err := db.ClaimTaskExecution(ctx, created.Task.ID, secondToken, 5*time.Minute); err != nil { + t.Fatalf("take over expired lease: %v", err) + } + if _, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: firstToken, Code: "old_worker", Message: "old"}); !errors.Is(err, ErrTaskExecutionLeaseLost) { + t.Fatalf("old worker terminal error=%v", err) + } + finished, err := db.FinishTaskFailure(ctx, FinishTaskFailureInput{TaskID: created.Task.ID, ExecutionToken: secondToken, Code: "new_worker", Message: "new"}) + if err != nil || finished.ErrorCode != "new_worker" { + t.Fatalf("new worker terminal task=%+v err=%v", finished, err) + } +} + +func TestBillingSettlementStaleTakeoverDebitsExactlyOnce(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-settle-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID} + if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{GatewayUserID: gatewayUserID, Currency: "resource", Balance: 10, Reason: "billing v2 test"}); err != nil { + t.Fatal(err) + } + created, err := db.CreateTask(ctx, CreateTaskInput{Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", Request: map[string]any{"model": "billing-v2-model"}}, user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid`, gatewayUserID) + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID) + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid`, gatewayUserID) + }) + token := uuid.NewString() + claimed, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute) + if err != nil { + t.Fatal(err) + } + amount := "1.000000001" + reservations, err := db.ReserveTaskBilling(ctx, claimed, user, []any{map[string]any{"currency": "resource", "amount": amount}}, map[string]any{ + "pricingVersion": "effective-pricing-v2", "reservationAmount": amount, "currency": "resource", "requestFingerprint": "billing-v2-test", + }) + if err != nil || len(reservations) != 1 { + t.Fatalf("reserve=%+v err=%v", reservations, err) + } + if _, err := db.FinishTaskSuccess(ctx, FinishTaskSuccessInput{ + TaskID: created.ID, ExecutionToken: token, Result: map[string]any{"ok": true}, + FinalChargeAmountText: amount, BillingCurrency: "resource", + PricingSnapshot: map[string]any{"pricingVersion": "effective-pricing-v2"}, + }); err != nil { + t.Fatal(err) + } + + firstClaims, err := db.ClaimBillingSettlements(ctx, "worker-one", BillingSettlementBatchSize, BillingSettlementLockTimeout) + if err != nil { + t.Fatal(err) + } + first := settlementForTask(t, firstClaims, created.ID) + if _, err := db.pool.Exec(ctx, `UPDATE settlement_outbox SET locked_at=now()-interval '3 minutes' WHERE id=$1::uuid`, first.ID); err != nil { + t.Fatal(err) + } + secondClaims, err := db.ClaimBillingSettlements(ctx, "worker-two", BillingSettlementBatchSize, BillingSettlementLockTimeout) + if err != nil { + t.Fatal(err) + } + second := settlementForTask(t, secondClaims, created.ID) + if err := db.ProcessBillingSettlement(ctx, first); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("stale settlement lock error=%v", err) + } + if err := db.ProcessBillingSettlement(ctx, second); err != nil { + t.Fatalf("process takeover: %v", err) + } + + var walletExact bool + if err := db.pool.QueryRow(ctx, ` +SELECT balance = 8.999999999::numeric + AND frozen_balance = 0::numeric + AND total_spent = 1.000000001::numeric +FROM gateway_wallet_accounts +WHERE gateway_user_id=$1::uuid AND currency='resource'`, gatewayUserID).Scan(&walletExact); err != nil { + t.Fatal(err) + } + if !walletExact { + t.Fatal("wallet amounts did not preserve nine-decimal settlement") + } + var billingTransactions int + if err := db.pool.QueryRow(ctx, `SELECT count(*) FROM gateway_wallet_transactions WHERE reference_id=$1 AND transaction_type='task_billing'`, created.ID).Scan(&billingTransactions); err != nil { + t.Fatal(err) + } + if billingTransactions != 1 { + t.Fatalf("task billing transactions=%d", billingTransactions) + } + if _, err := db.pool.Exec(ctx, ` +DELETE FROM gateway_wallet_accounts +WHERE gateway_user_id=$1::uuid AND currency='resource'`, gatewayUserID); err == nil { + t.Fatal("wallet account with audit transactions must not be deletable") + } + if err := db.pool.QueryRow(ctx, ` +SELECT count(*) FROM gateway_wallet_transactions +WHERE gateway_user_id=$1::uuid AND reference_id=$2`, gatewayUserID, created.ID).Scan(&billingTransactions); err != nil { + t.Fatal(err) + } + if billingTransactions == 0 { + t.Fatal("wallet audit transactions were lost after rejected account deletion") + } + settled, err := db.GetTask(ctx, created.ID) + if err != nil || settled.BillingStatus != "settled" { + t.Fatalf("settled task=%+v err=%v", settled, err) + } +} + +func TestReleaseTaskBillingReservationsPreservesNineDecimalPlaces(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-release-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID} + if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{ + GatewayUserID: gatewayUserID, Currency: "resource", Balance: 1, Reason: "billing v2 release test", + }); err != nil { + t.Fatal(err) + } + created, err := db.CreateTask(ctx, CreateTaskInput{ + Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", + Request: map[string]any{"model": "billing-v2-model"}, + }, user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx := context.Background() + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid`, gatewayUserID) + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID) + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid`, gatewayUserID) + }) + amount := "0.000000001" + reservations, err := db.ReserveTaskBilling(ctx, created, user, nil, map[string]any{ + "pricingVersion": "effective-pricing-v2", "reservationAmount": amount, "currency": "resource", + }) + if err != nil || len(reservations) != 1 { + t.Fatalf("reserve=%+v err=%v", reservations, err) + } + if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{ + GatewayUserID: gatewayUserID, Currency: "resource", BalanceText: "0", Reason: "must not cross frozen balance", + }); !errors.Is(err, ErrBalanceBelowFrozen) { + t.Fatalf("balance below frozen error=%v", err) + } + if err := db.ReleaseTaskBillingReservations(ctx, reservations, "integration_test"); err != nil { + t.Fatal(err) + } + if _, err := db.SetUserWalletBalance(ctx, WalletBalanceAdjustmentInput{ + GatewayUserID: gatewayUserID, Currency: "resource", BalanceText: "0.123456789", Reason: "exact adjustment", + }); err != nil { + t.Fatal(err) + } + if _, err := db.RechargeUserWalletBalance(ctx, WalletRechargeInput{ + GatewayUserID: gatewayUserID, Currency: "resource", AmountText: "0.000000001", Reason: "exact recharge", + }); err != nil { + t.Fatal(err) + } + var exact bool + if err := db.pool.QueryRow(ctx, ` +SELECT account.balance = 0.123456790::numeric + AND account.frozen_balance = 0::numeric + AND task.reservation_amount = 0::numeric + AND task.billing_status = 'not_started' + AND EXISTS ( + SELECT 1 FROM gateway_wallet_transactions transaction + WHERE transaction.reference_id = task.id::text + AND transaction.transaction_type = 'release' + AND transaction.amount = 0.000000001::numeric + ) +FROM gateway_wallet_accounts account +JOIN gateway_tasks task ON task.gateway_user_id = account.gateway_user_id +WHERE task.id=$1::uuid AND account.currency='resource'`, created.ID).Scan(&exact); err != nil { + t.Fatal(err) + } + if !exact { + t.Fatal("nine-decimal reservation was not released exactly") + } +} + +func billingV2IntegrationStore(t *testing.T) *Store { + t.Helper() + databaseURL := strings.TrimSpace(os.Getenv("AI_GATEWAY_TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("set AI_GATEWAY_TEST_DATABASE_URL to run billing v2 PostgreSQL integration tests") + } + db, err := Connect(context.Background(), databaseURL) + if err != nil { + t.Fatal(err) + } + var databaseName string + if err := db.pool.QueryRow(context.Background(), `SELECT current_database()`).Scan(&databaseName); err != nil { + db.Close() + t.Fatal(err) + } + if !strings.Contains(strings.ToLower(databaseName), "test") { + db.Close() + t.Fatalf("refusing to use non-test database %q", databaseName) + } + t.Cleanup(db.Close) + return db +} + +func settlementForTask(t *testing.T, items []BillingSettlement, taskID string) BillingSettlement { + t.Helper() + for _, item := range items { + if item.TaskID == taskID { + return item + } + } + t.Fatalf("settlement for task %s not claimed", taskID) + return BillingSettlement{} +} diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index c364834..b3d7f93 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -47,16 +47,21 @@ func normalizeAPIKeyScopes(scopes []string) []string { } var ( - ErrInvalidCredentials = errors.New("invalid account or password") - ErrInvalidInvitation = errors.New("invalid or expired invitation code") - ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty") - ErrAccessRuleResourceDenied = errors.New("access rule resource is not available") - ErrInsufficientWalletBalance = errors.New("insufficient wallet balance") - ErrLocalUserRequired = errors.New("local gateway user is required") - ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged") - ErrProtectedDefault = errors.New("protected default resource cannot be deleted") - ErrUserAlreadyExists = errors.New("user already exists") - ErrWeakPassword = errors.New("password must be at least 8 characters") + ErrInvalidCredentials = errors.New("invalid account or password") + ErrInvalidInvitation = errors.New("invalid or expired invitation code") + ErrInvalidAPIKeyScopes = errors.New("api key scopes must not be empty") + ErrAccessRuleResourceDenied = errors.New("access rule resource is not available") + ErrInsufficientWalletBalance = errors.New("insufficient wallet balance") + ErrLocalUserRequired = errors.New("local gateway user is required") + ErrWalletBalanceUnchanged = errors.New("wallet balance unchanged") + ErrBalanceBelowFrozen = errors.New("wallet balance cannot be below frozen balance") + ErrInvalidWalletAmount = errors.New("wallet amount must be a decimal with at most nine fractional digits") + ErrIdempotencyKeyReused = errors.New("idempotency key was reused for a different request") + ErrTaskExecutionLeaseUnavailable = errors.New("task execution lease is unavailable") + ErrTaskExecutionLeaseLost = errors.New("task execution lease was lost") + ErrProtectedDefault = errors.New("protected default resource cannot be deleted") + ErrUserAlreadyExists = errors.New("user already exists") + ErrWeakPassword = errors.New("password must be at least 8 characters") ) func Connect(ctx context.Context, databaseURL string) (*Store, error) { @@ -414,64 +419,81 @@ type RateLimitWindow struct { } type CreateTaskInput struct { - Kind string `json:"kind"` - Model string `json:"model"` - RunMode string `json:"runMode"` - Async bool `json:"async"` - Request map[string]any `json:"request"` - ConversationID string `json:"conversationId"` - NewMessageCount int `json:"newMessageCount"` - MessageRefs []TaskMessageRefInput `json:"messageRefs"` + Kind string `json:"kind"` + Model string `json:"model"` + RunMode string `json:"runMode"` + Async bool `json:"async"` + Request map[string]any `json:"request"` + ConversationID string `json:"conversationId"` + NewMessageCount int `json:"newMessageCount"` + MessageRefs []TaskMessageRefInput `json:"messageRefs"` + IdempotencyKeyHash string `json:"-"` + IdempotencyRequestHash string `json:"-"` +} + +type CreateTaskResult struct { + Task GatewayTask + Replayed bool } type GatewayTask struct { - ID string `json:"id"` - Kind string `json:"kind"` - RunMode string `json:"runMode"` - UserID string `json:"userId"` - GatewayUserID string `json:"gatewayUserId,omitempty"` - UserSource string `json:"userSource,omitempty"` - GatewayTenantID string `json:"gatewayTenantId,omitempty"` - TenantID string `json:"tenantId,omitempty"` - TenantKey string `json:"tenantKey,omitempty"` - APIKeyID string `json:"apiKeyId,omitempty"` - APIKeyName string `json:"apiKeyName,omitempty"` - APIKeyPrefix string `json:"apiKeyPrefix,omitempty"` - UserGroupID string `json:"userGroupId,omitempty"` - UserGroupKey string `json:"userGroupKey,omitempty"` - Model string `json:"model"` - ModelType string `json:"modelType,omitempty"` - RequestedModel string `json:"requestedModel,omitempty"` - ResolvedModel string `json:"resolvedModel,omitempty"` - RequestID string `json:"requestId,omitempty"` - ConversationID string `json:"conversationId,omitempty"` - NewMessageCount int `json:"newMessageCount,omitempty"` - Request map[string]any `json:"request,omitempty"` - AsyncMode bool `json:"asyncMode"` - RiverJobID int64 `json:"riverJobId,omitempty"` - Status string `json:"status"` - Cancellable *bool `json:"cancellable,omitempty"` - Submitted *bool `json:"submitted,omitempty"` - Message string `json:"message,omitempty"` - AttemptCount int `json:"attemptCount"` - RemoteTaskID string `json:"remoteTaskId,omitempty"` - RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"` - Result map[string]any `json:"result,omitempty"` - Billings []any `json:"billings,omitempty"` - Usage map[string]any `json:"usage"` - Metrics map[string]any `json:"metrics"` - BillingSummary map[string]any `json:"billingSummary"` - FinalChargeAmount float64 `json:"finalChargeAmount"` - ResponseStartedAt string `json:"responseStartedAt,omitempty"` - ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` - ResponseDurationMS int64 `json:"responseDurationMs"` - FinishedAt string `json:"finishedAt,omitempty"` - Error string `json:"error,omitempty"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - Attempts []TaskAttempt `json:"attempts,omitempty"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ID string `json:"id"` + Kind string `json:"kind"` + RunMode string `json:"runMode"` + UserID string `json:"userId"` + GatewayUserID string `json:"gatewayUserId,omitempty"` + UserSource string `json:"userSource,omitempty"` + GatewayTenantID string `json:"gatewayTenantId,omitempty"` + TenantID string `json:"tenantId,omitempty"` + TenantKey string `json:"tenantKey,omitempty"` + APIKeyID string `json:"apiKeyId,omitempty"` + APIKeyName string `json:"apiKeyName,omitempty"` + APIKeyPrefix string `json:"apiKeyPrefix,omitempty"` + UserGroupID string `json:"userGroupId,omitempty"` + UserGroupKey string `json:"userGroupKey,omitempty"` + Model string `json:"model"` + ModelType string `json:"modelType,omitempty"` + RequestedModel string `json:"requestedModel,omitempty"` + ResolvedModel string `json:"resolvedModel,omitempty"` + RequestID string `json:"requestId,omitempty"` + ConversationID string `json:"conversationId,omitempty"` + NewMessageCount int `json:"newMessageCount,omitempty"` + Request map[string]any `json:"request,omitempty"` + AsyncMode bool `json:"asyncMode"` + RiverJobID int64 `json:"riverJobId,omitempty"` + Status string `json:"status"` + Cancellable *bool `json:"cancellable,omitempty"` + Submitted *bool `json:"submitted,omitempty"` + Message string `json:"message,omitempty"` + AttemptCount int `json:"attemptCount"` + RemoteTaskID string `json:"remoteTaskId,omitempty"` + RemoteTaskPayload map[string]any `json:"remoteTaskPayload,omitempty"` + Result map[string]any `json:"result,omitempty"` + Billings []any `json:"billings,omitempty"` + Usage map[string]any `json:"usage"` + Metrics map[string]any `json:"metrics"` + BillingSummary map[string]any `json:"billingSummary"` + FinalChargeAmount float64 `json:"finalChargeAmount"` + BillingVersion string `json:"billingVersion"` + BillingStatus string `json:"billingStatus"` + BillingCurrency string `json:"billingCurrency"` + PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` + RequestFingerprint string `json:"requestFingerprint,omitempty"` + ReservationAmount float64 `json:"reservationAmount"` + ExecutionToken string `json:"-"` + ExecutionLeaseUntil string `json:"executionLeaseExpiresAt,omitempty"` + BillingUpdatedAt string `json:"billingUpdatedAt,omitempty"` + BillingSettledAt string `json:"billingSettledAt,omitempty"` + ResponseStartedAt string `json:"responseStartedAt,omitempty"` + ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` + ResponseDurationMS int64 `json:"responseDurationMs"` + FinishedAt string `json:"finishedAt,omitempty"` + Error string `json:"error,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + Attempts []TaskAttempt `json:"attempts,omitempty"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` } const gatewayTaskColumns = ` @@ -485,7 +507,11 @@ request, COALESCE(async_mode, false), COALESCE(river_job_id, 0), status, COALESC COALESCE(remote_task_id, ''), COALESCE(remote_task_payload, '{}'::jsonb), COALESCE(result, '{}'::jsonb), COALESCE(billings, '[]'::jsonb), COALESCE(usage, '{}'::jsonb), COALESCE(metrics, '{}'::jsonb), COALESCE(billing_summary, '{}'::jsonb), -COALESCE(final_charge_amount, 0)::float8, COALESCE(response_started_at::text, ''), +COALESCE(final_charge_amount, 0)::float8, billing_version, billing_status, billing_currency, +COALESCE(pricing_snapshot, '{}'::jsonb), COALESCE(request_fingerprint, ''), +COALESCE(reservation_amount, 0)::float8, COALESCE(execution_token::text, ''), +COALESCE(execution_lease_expires_at::text, ''), COALESCE(billing_updated_at::text, ''), +COALESCE(billing_settled_at::text, ''), COALESCE(response_started_at::text, ''), COALESCE(response_finished_at::text, ''), COALESCE(response_duration_ms, 0), COALESCE(error, ''), COALESCE(error_code, ''), COALESCE(error_message, ''), created_at, updated_at, COALESCE(finished_at::text, '')` @@ -505,35 +531,39 @@ type TaskEvent struct { } type TaskAttempt struct { - ID string `json:"id"` - TaskID string `json:"taskId"` - AttemptNo int `json:"attemptNo"` - PlatformID string `json:"platformId,omitempty"` - PlatformName string `json:"platformName,omitempty"` - Provider string `json:"provider,omitempty"` - PlatformModelID string `json:"platformModelId,omitempty"` - ModelName string `json:"modelName,omitempty"` - ProviderModelName string `json:"providerModelName,omitempty"` - ModelAlias string `json:"modelAlias,omitempty"` - ModelType string `json:"modelType,omitempty"` - ClientID string `json:"clientId,omitempty"` - QueueKey string `json:"queueKey"` - Status string `json:"status"` - Retryable bool `json:"retryable"` - Simulated bool `json:"simulated"` - RequestID string `json:"requestId,omitempty"` - StatusCode int `json:"statusCode,omitempty"` - Usage map[string]any `json:"usage,omitempty"` - Metrics map[string]any `json:"metrics,omitempty"` - RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"` - ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"` - ResponseStartedAt string `json:"responseStartedAt,omitempty"` - ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` - ResponseDurationMS int64 `json:"responseDurationMs"` - ErrorCode string `json:"errorCode,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - StartedAt time.Time `json:"startedAt"` - FinishedAt string `json:"finishedAt,omitempty"` + ID string `json:"id"` + TaskID string `json:"taskId"` + AttemptNo int `json:"attemptNo"` + PlatformID string `json:"platformId,omitempty"` + PlatformName string `json:"platformName,omitempty"` + Provider string `json:"provider,omitempty"` + PlatformModelID string `json:"platformModelId,omitempty"` + ModelName string `json:"modelName,omitempty"` + ProviderModelName string `json:"providerModelName,omitempty"` + ModelAlias string `json:"modelAlias,omitempty"` + ModelType string `json:"modelType,omitempty"` + ClientID string `json:"clientId,omitempty"` + QueueKey string `json:"queueKey"` + Status string `json:"status"` + Retryable bool `json:"retryable"` + Simulated bool `json:"simulated"` + RequestID string `json:"requestId,omitempty"` + StatusCode int `json:"statusCode,omitempty"` + Usage map[string]any `json:"usage,omitempty"` + Metrics map[string]any `json:"metrics,omitempty"` + RequestSnapshot map[string]any `json:"requestSnapshot,omitempty"` + ResponseSnapshot map[string]any `json:"responseSnapshot,omitempty"` + ResponseStartedAt string `json:"responseStartedAt,omitempty"` + ResponseFinishedAt string `json:"responseFinishedAt,omitempty"` + ResponseDurationMS int64 `json:"responseDurationMs"` + PricingSnapshot map[string]any `json:"pricingSnapshot,omitempty"` + RequestFingerprint string `json:"requestFingerprint,omitempty"` + UpstreamSubmissionStatus string `json:"upstreamSubmissionStatus"` + UpstreamSubmissionUpdatedAt string `json:"upstreamSubmissionUpdatedAt,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + ErrorMessage string `json:"errorMessage,omitempty"` + StartedAt time.Time `json:"startedAt"` + FinishedAt string `json:"finishedAt,omitempty"` } type TaskParamPreprocessingLog struct { @@ -1716,6 +1746,9 @@ ON CONFLICT (gateway_user_id, currency) DO NOTHING`, ); err != nil { return GatewayUser{}, err } + if err := ensureWalletAccountAuditGuard(ctx, tx, user.ID, "resource"); err != nil { + return GatewayUser{}, err + } if err := tx.Commit(ctx); err != nil { return GatewayUser{}, err } @@ -1826,6 +1859,11 @@ ORDER BY window_start DESC, scope_type ASC, scope_key ASC, metric ASC`) } func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *auth.User) (GatewayTask, error) { + result, err := s.CreateTaskIdempotent(ctx, input, user) + return result.Task, err +} + +func (s *Store) CreateTaskIdempotent(ctx context.Context, input CreateTaskInput, user *auth.User) (CreateTaskResult, error) { requestBody, _ := json.Marshal(input.Request) runMode := normalizeRunMode(input.RunMode, input.Request) status := "queued" @@ -1834,7 +1872,7 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut tx, err := s.pool.Begin(ctx) if err != nil { - return GatewayTask{}, err + return CreateTaskResult{}, err } defer tx.Rollback(ctx) @@ -1842,33 +1880,62 @@ func (s *Store) CreateTask(ctx context.Context, input CreateTaskInput, user *aut INSERT INTO gateway_tasks ( kind, run_mode, user_id, gateway_user_id, user_source, gateway_tenant_id, tenant_id, tenant_key, api_key_id, api_key_name, api_key_prefix, user_group_id, user_group_key, - model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count, finished_at + model, requested_model, request, async_mode, status, result, billings, conversation_id, new_message_count, + idempotency_key_hash, idempotency_request_hash, finished_at ) - VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, CASE WHEN $22 THEN now() ELSE NULL END) + VALUES ($1, $2, $3, NULLIF($4, '')::uuid, COALESCE(NULLIF($5, ''), 'gateway'), NULLIF($6, '')::uuid, NULLIF($7, ''), NULLIF($8, ''), NULLIF($9, ''), NULLIF($10, ''), NULLIF($11, ''), NULLIF($12, '')::uuid, NULLIF($13, ''), $14, $14, $15, $16, $17, $18::jsonb, $19::jsonb, NULLIF($20, '')::uuid, $21, NULLIF($22, ''), NULLIF($23, ''), NULL) + ON CONFLICT (user_id, idempotency_key_hash) WHERE idempotency_key_hash IS NOT NULL DO NOTHING RETURNING `+gatewayTaskColumns, - input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, false, + input.Kind, runMode, user.ID, user.GatewayUserID, user.Source, user.GatewayTenantID, user.TenantID, user.TenantKey, user.APIKeyID, user.APIKeyName, user.APIKeyPrefix, user.UserGroupID, user.UserGroupKey, input.Model, requestBody, input.Async, status, resultBody, billingsBody, input.ConversationID, input.NewMessageCount, strings.TrimSpace(input.IdempotencyKeyHash), strings.TrimSpace(input.IdempotencyRequestHash), )) + replayed := false + if errors.Is(err, pgx.ErrNoRows) && strings.TrimSpace(input.IdempotencyKeyHash) != "" { + var existingRequestHash string + if err := tx.QueryRow(ctx, ` +SELECT COALESCE(idempotency_request_hash, '') +FROM gateway_tasks +WHERE user_id = $1 AND idempotency_key_hash = $2 +FOR UPDATE`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash)).Scan(&existingRequestHash); err != nil { + return CreateTaskResult{}, err + } + if existingRequestHash != strings.TrimSpace(input.IdempotencyRequestHash) { + return CreateTaskResult{}, ErrIdempotencyKeyReused + } + task, err = scanGatewayTask(tx.QueryRow(ctx, ` +SELECT `+gatewayTaskColumns+` +FROM gateway_tasks +WHERE user_id = $1 AND idempotency_key_hash = $2`, user.ID, strings.TrimSpace(input.IdempotencyKeyHash))) + replayed = true + } if err != nil { - return GatewayTask{}, err + return CreateTaskResult{}, err } - if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil { - return GatewayTask{}, err - } - events := taskEventsForCreate(task.ID, runMode, status, nil) - for _, event := range events { - payload, _ := json.Marshal(event.Payload) - if _, err := tx.Exec(ctx, ` + if !replayed { + if err := insertTaskMessageRefs(ctx, tx, task.ID, input.MessageRefs); err != nil { + return CreateTaskResult{}, err + } + events := taskEventsForCreate(task.ID, runMode, status, nil) + for _, event := range events { + payload, _ := json.Marshal(event.Payload) + if _, err := tx.Exec(ctx, ` INSERT INTO gateway_task_events (task_id, seq, event_type, status, phase, progress, message, payload, simulated) VALUES ($1::uuid, $2, $3::text, NULLIF($4::text, ''), NULLIF($5::text, ''), $6, NULLIF($7::text, ''), $8::jsonb, $9)`, - task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated, - ); err != nil { - return GatewayTask{}, err + task.ID, event.Seq, event.EventType, event.Status, event.Phase, event.Progress, event.Message, string(payload), event.Simulated, + ); err != nil { + return CreateTaskResult{}, err + } } } if err := tx.Commit(ctx); err != nil { - return GatewayTask{}, err + return CreateTaskResult{}, err } - return task, nil + if replayed { + task, err = s.GetTask(ctx, task.ID) + if err != nil { + return CreateTaskResult{}, err + } + } + return CreateTaskResult{Task: task, Replayed: replayed}, nil } func (s *Store) GetTask(ctx context.Context, taskID string) (GatewayTask, error) { @@ -1900,6 +1967,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { var usageBytes []byte var metricsBytes []byte var billingSummaryBytes []byte + var pricingSnapshotBytes []byte var remoteTaskPayloadBytes []byte if err := scanner.Scan( &task.ID, @@ -1936,6 +2004,16 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { &metricsBytes, &billingSummaryBytes, &task.FinalChargeAmount, + &task.BillingVersion, + &task.BillingStatus, + &task.BillingCurrency, + &pricingSnapshotBytes, + &task.RequestFingerprint, + &task.ReservationAmount, + &task.ExecutionToken, + &task.ExecutionLeaseUntil, + &task.BillingUpdatedAt, + &task.BillingSettledAt, &task.ResponseStartedAt, &task.ResponseFinishedAt, &task.ResponseDurationMS, @@ -1955,6 +2033,7 @@ func scanGatewayTask(scanner taskScanner) (GatewayTask, error) { task.Usage = decodeObject(usageBytes) task.Metrics = decodeObject(metricsBytes) task.BillingSummary = decodeObject(billingSummaryBytes) + task.PricingSnapshot = decodeObject(pricingSnapshotBytes) return task, nil } diff --git a/apps/api/internal/store/runtime_types.go b/apps/api/internal/store/runtime_types.go index c4c4b5f..cd2e7ed 100644 --- a/apps/api/internal/store/runtime_types.go +++ b/apps/api/internal/store/runtime_types.go @@ -223,21 +223,24 @@ type RateLimitResult struct { } type CreateTaskAttemptInput struct { - TaskID string - AttemptNo int - PlatformID string - PlatformModelID string - ClientID string - QueueKey string - Status string - Simulated bool - RequestSnapshot map[string]any - Metrics map[string]any + TaskID string + AttemptNo int + PlatformID string + PlatformModelID string + ClientID string + QueueKey string + Status string + Simulated bool + RequestSnapshot map[string]any + Metrics map[string]any + PricingSnapshot map[string]any + RequestFingerprint string } type AsyncTaskQueueItem struct { - ID string - Priority int + ID string + Priority int + NextRunAt time.Time } type FinishTaskAttemptInput struct { @@ -256,15 +259,37 @@ type FinishTaskAttemptInput struct { } type FinishTaskSuccessInput struct { + TaskID string + ExecutionToken string + AttemptID string + Result map[string]any + Billings []any + RequestID string + ResolvedModel string + Usage map[string]any + Metrics map[string]any + BillingSummary map[string]any + FinalChargeAmount float64 + FinalChargeAmountText string + BillingCurrency string + PricingSnapshot map[string]any + RequestFingerprint string + ResponseStartedAt time.Time + ResponseFinishedAt time.Time + ResponseDurationMS int64 +} + +type FinishTaskManualReviewInput struct { TaskID string + ExecutionToken string + AttemptID string + TaskStatus string + Code string + Message string Result map[string]any - Billings []any RequestID string - ResolvedModel string - Usage map[string]any - Metrics map[string]any - BillingSummary map[string]any - FinalChargeAmount float64 + PricingSnapshot map[string]any + RequestFingerprint string ResponseStartedAt time.Time ResponseFinishedAt time.Time ResponseDurationMS int64 @@ -272,6 +297,7 @@ type FinishTaskSuccessInput struct { type FinishTaskFailureInput struct { TaskID string + ExecutionToken string Code string Message string Result map[string]any diff --git a/apps/api/internal/store/tasks_runtime.go b/apps/api/internal/store/tasks_runtime.go index f4009a3..0b55e76 100644 --- a/apps/api/internal/store/tasks_runtime.go +++ b/apps/api/internal/store/tasks_runtime.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/json" + "errors" "fmt" "strconv" "strings" @@ -152,18 +153,71 @@ func nullableTaskListTime(value *time.Time) any { return *value } -func (s *Store) MarkTaskRunning(ctx context.Context, taskID string, modelType string, normalizedRequest map[string]any) error { - normalizedJSON, _ := json.Marshal(emptyObjectIfNil(normalizedRequest)) - _, err := s.pool.Exec(ctx, ` +func (s *Store) ClaimTaskExecution(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) (GatewayTask, error) { + if leaseTTL <= 0 { + leaseTTL = 5 * time.Minute + } + task, err := scanGatewayTask(s.pool.QueryRow(ctx, ` UPDATE gateway_tasks SET status = 'running', - model_type = NULLIF($2::text, ''), - normalized_request = $3::jsonb, + execution_token = $2::uuid, + execution_lease_expires_at = now() + ($3::int * interval '1 second'), locked_at = now(), heartbeat_at = now(), updated_at = now() -WHERE id = $1::uuid`, taskID, modelType, string(normalizedJSON)) - return err +WHERE id = $1::uuid + AND ( + (status = 'queued' AND next_run_at <= now()) + OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now())) + ) +RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second))) + if errors.Is(err, pgx.ErrNoRows) { + return GatewayTask{}, ErrTaskExecutionLeaseUnavailable + } + return task, err +} + +func (s *Store) RenewTaskExecutionLease(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) error { + if leaseTTL <= 0 { + leaseTTL = 5 * time.Minute + } + tag, err := s.pool.Exec(ctx, ` +UPDATE gateway_tasks +SET execution_lease_expires_at = now() + ($3::int * interval '1 second'), + heartbeat_at = now(), + updated_at = now() +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $2::uuid`, taskID, executionToken, int(leaseTTL/time.Second)) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } + return nil +} + +func (s *Store) MarkTaskRunning(ctx context.Context, taskID string, executionToken string, modelType string, normalizedRequest map[string]any) error { + normalizedJSON, _ := json.Marshal(emptyObjectIfNil(normalizedRequest)) + tag, err := s.pool.Exec(ctx, ` +UPDATE gateway_tasks +SET status = 'running', + model_type = NULLIF($3::text, ''), + normalized_request = $4::jsonb, + locked_at = now(), + heartbeat_at = now(), + updated_at = now() +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $2::uuid`, taskID, executionToken, modelType, string(normalizedJSON)) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } + return nil } func (s *Store) ClaimAsyncQueuedTask(ctx context.Context, workerID string) (GatewayTask, error) { @@ -189,7 +243,7 @@ WHERE t.id = picked.task_id RETURNING `+gatewayTaskColumns, workerID)) } -func (s *Store) RequeueTask(ctx context.Context, taskID string, delay time.Duration, queueKey string) (GatewayTask, error) { +func (s *Store) RequeueTask(ctx context.Context, taskID string, executionToken string, delay time.Duration, queueKey string) (GatewayTask, error) { if delay < time.Second { delay = time.Second } @@ -203,14 +257,18 @@ SET status = 'queued', locked_by = NULL, locked_at = NULL, heartbeat_at = NULL, - next_run_at = $2::timestamptz, - queue_key = COALESCE(NULLIF($3::text, ''), queue_key), + execution_token = NULL, + execution_lease_expires_at = NULL, + next_run_at = $3::timestamptz, + queue_key = COALESCE(NULLIF($4::text, ''), queue_key), error = NULL, error_code = NULL, error_message = NULL, updated_at = now() WHERE id = $1::uuid -RETURNING `+gatewayTaskColumns, taskID, nextRunAt, strings.TrimSpace(queueKey))) + AND status = 'running' + AND execution_token = $2::uuid +RETURNING `+gatewayTaskColumns, taskID, executionToken, nextRunAt, strings.TrimSpace(queueKey))) } func (s *Store) SetTaskRiverJobID(ctx context.Context, taskID string, riverJobID int64) error { @@ -225,25 +283,32 @@ WHERE id = $1::uuid`, taskID, riverJobID) return err } -func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, attemptID string, remoteTaskID string, payload map[string]any) error { +func (s *Store) SetTaskRemoteTask(ctx context.Context, taskID string, executionToken string, attemptID string, remoteTaskID string, payload map[string]any) error { payloadJSON, _ := json.Marshal(emptyObjectIfNil(payload)) return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { - if _, err := tx.Exec(ctx, ` + tag, err := tx.Exec(ctx, ` UPDATE gateway_tasks -SET remote_task_id = NULLIF($2::text, ''), - remote_task_payload = $3::jsonb, +SET remote_task_id = NULLIF($3::text, ''), + remote_task_payload = $4::jsonb, updated_at = now() -WHERE id = $1::uuid`, +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $2::uuid`, taskID, + executionToken, remoteTaskID, string(payloadJSON), - ); err != nil { + ) + if err != nil { return err } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } if strings.TrimSpace(attemptID) == "" { return nil } - _, err := tx.Exec(ctx, ` + _, err = tx.Exec(ctx, ` UPDATE gateway_task_attempts SET remote_task_id = NULLIF($2::text, ''), response_snapshot = COALESCE(response_snapshot, '{}'::jsonb) || jsonb_build_object('remote_task_payload', $3::jsonb) @@ -256,6 +321,20 @@ WHERE id = $1::uuid`, }) } +func (s *Store) SetAttemptUpstreamSubmissionStatus(ctx context.Context, attemptID string, status string) error { + switch status { + case "not_submitted", "submitting", "response_received": + default: + return fmt.Errorf("invalid upstream submission status %q", status) + } + _, err := s.pool.Exec(ctx, ` +UPDATE gateway_task_attempts +SET upstream_submission_status = $2, + upstream_submission_updated_at = now() +WHERE id = $1::uuid`, attemptID, status) + return err +} + func (s *Store) CancelQueuedTask(ctx context.Context, taskID string, message string) (GatewayTask, bool, error) { message = strings.TrimSpace(message) if message == "" { @@ -293,10 +372,13 @@ func (s *Store) ListRecoverableAsyncTasks(ctx context.Context, limit int) ([]Asy limit = 500 } rows, err := s.pool.Query(ctx, ` -SELECT id::text, priority +SELECT id::text, priority, next_run_at FROM gateway_tasks WHERE async_mode = true - AND status IN ('queued', 'running') + AND ( + status = 'queued' + OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now())) + ) ORDER BY priority ASC, created_at ASC LIMIT $1`, limit) if err != nil { @@ -306,7 +388,7 @@ LIMIT $1`, limit) items := make([]AsyncTaskQueueItem, 0) for rows.Next() { var item AsyncTaskQueueItem - if err := rows.Scan(&item.ID, &item.Priority); err != nil { + if err := rows.Scan(&item.ID, &item.Priority, &item.NextRunAt); err != nil { return nil, err } items = append(items, item) @@ -320,6 +402,7 @@ LIMIT $1`, limit) func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptInput) (string, error) { requestJSON, _ := json.Marshal(emptyObjectIfNil(input.RequestSnapshot)) metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) + pricingJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot)) tx, err := s.pool.Begin(ctx) if err != nil { return "", err @@ -330,11 +413,13 @@ func (s *Store) CreateTaskAttempt(ctx context.Context, input CreateTaskAttemptIn err = tx.QueryRow(ctx, ` INSERT INTO gateway_task_attempts ( task_id, attempt_no, platform_id, platform_model_id, client_id, queue_key, - status, simulated, request_snapshot, metrics + status, simulated, request_snapshot, metrics, pricing_snapshot, request_fingerprint, + upstream_submission_status, upstream_submission_updated_at ) VALUES ( $1::uuid, $2::int, NULLIF($3::text, '')::uuid, NULLIF($4::text, '')::uuid, NULLIF($5::text, ''), $6, - $7, $8, $9::jsonb, $10::jsonb + $7, $8, $9::jsonb, $10::jsonb, $11::jsonb, NULLIF($12, ''), + 'not_submitted', now() ) RETURNING id::text`, input.TaskID, @@ -347,6 +432,8 @@ RETURNING id::text`, input.Simulated, string(requestJSON), string(metricsJSON), + string(pricingJSON), + input.RequestFingerprint, ).Scan(&attemptID) if err != nil { return "", err @@ -489,6 +576,8 @@ SELECT a.id::text, a.task_id::text, a.attempt_no, a.request_snapshot, COALESCE(a.response_snapshot, '{}'::jsonb), COALESCE(a.response_started_at::text, ''), COALESCE(a.response_finished_at::text, ''), COALESCE(a.response_duration_ms, 0), COALESCE(a.error_code, ''), COALESCE(a.error_message, ''), + COALESCE(a.pricing_snapshot, '{}'::jsonb), COALESCE(a.request_fingerprint, ''), + a.upstream_submission_status, COALESCE(a.upstream_submission_updated_at::text, ''), a.started_at, COALESCE(a.finished_at::text, '') FROM gateway_task_attempts a LEFT JOIN integration_platforms p ON p.id = a.platform_id @@ -518,6 +607,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { var metricsBytes []byte var requestBytes []byte var responseBytes []byte + var pricingSnapshotBytes []byte if err := scanner.Scan( &item.ID, &item.TaskID, @@ -544,6 +634,10 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { &item.ResponseDurationMS, &item.ErrorCode, &item.ErrorMessage, + &pricingSnapshotBytes, + &item.RequestFingerprint, + &item.UpstreamSubmissionStatus, + &item.UpstreamSubmissionUpdatedAt, &item.StartedAt, &item.FinishedAt, ); err != nil { @@ -553,6 +647,7 @@ func scanTaskAttempt(scanner taskScanner) (TaskAttempt, error) { item.Metrics = decodeObject(metricsBytes) item.RequestSnapshot = decodeObject(requestBytes) item.ResponseSnapshot = decodeObject(responseBytes) + item.PricingSnapshot = decodeObject(pricingSnapshotBytes) enrichTaskAttemptFromMetrics(&item) return item, nil } @@ -670,7 +765,41 @@ func (s *Store) FinishTaskSuccess(ctx context.Context, input FinishTaskSuccessIn usageJSON, _ := json.Marshal(emptyObjectIfNil(input.Usage)) metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) billingSummaryJSON, _ := json.Marshal(emptyObjectIfNil(input.BillingSummary)) - if _, err := s.pool.Exec(ctx, ` + pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot)) + finalChargeAmount := strings.TrimSpace(input.FinalChargeAmountText) + if finalChargeAmount == "" { + finalChargeAmount = strconv.FormatFloat(input.FinalChargeAmount, 'f', 9, 64) + } + currency := normalizeWalletCurrency(input.BillingCurrency) + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if strings.TrimSpace(input.AttemptID) != "" { + if _, err := tx.Exec(ctx, ` +UPDATE gateway_task_attempts +SET status = 'succeeded', + retryable = false, + request_id = NULLIF($2, ''), + usage = $3::jsonb, + metrics = $4::jsonb, + response_snapshot = $5::jsonb, + pricing_snapshot = $6::jsonb, + request_fingerprint = NULLIF($7, ''), + upstream_submission_status = 'response_received', + upstream_submission_updated_at = now(), + response_started_at = $8::timestamptz, + response_finished_at = $9::timestamptz, + response_duration_ms = $10, + error_code = NULL, + error_message = NULL, + finished_at = now() +WHERE id = $1::uuid`, + input.AttemptID, input.RequestID, string(usageJSON), string(metricsJSON), + string(resultJSON), string(pricingSnapshotJSON), input.RequestFingerprint, + nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS, + ); err != nil { + return err + } + } + tag, err := tx.Exec(ctx, ` UPDATE gateway_tasks SET status = 'succeeded', result = $2::jsonb, @@ -680,32 +809,142 @@ SET status = 'succeeded', usage = $6::jsonb, metrics = $7::jsonb, billing_summary = $8::jsonb, - final_charge_amount = $9, - response_started_at = $10::timestamptz, - response_finished_at = $11::timestamptz, - response_duration_ms = $12, + final_charge_amount = $9::numeric, + billing_version = 'effective-pricing-v2', + billing_status = CASE + WHEN run_mode = 'production' AND gateway_user_id IS NOT NULL THEN 'pending' + ELSE 'not_required' + END, + billing_currency = $10, + pricing_snapshot = $11::jsonb, + request_fingerprint = NULLIF($12, ''), + billing_updated_at = now(), + response_started_at = $13::timestamptz, + response_finished_at = $14::timestamptz, + response_duration_ms = $15, error = NULL, error_code = NULL, error_message = NULL, locked_by = NULL, locked_at = NULL, heartbeat_at = NULL, + execution_token = NULL, + execution_lease_expires_at = NULL, finished_at = now(), updated_at = now() -WHERE id = $1::uuid`, - input.TaskID, - string(resultJSON), - string(billingsJSON), - input.RequestID, - input.ResolvedModel, - string(usageJSON), - string(metricsJSON), - string(billingSummaryJSON), - input.FinalChargeAmount, - nullableTime(input.ResponseStartedAt), - nullableTime(input.ResponseFinishedAt), - input.ResponseDurationMS, - ); err != nil { +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $16::uuid`, + input.TaskID, + string(resultJSON), + string(billingsJSON), + input.RequestID, + input.ResolvedModel, + string(usageJSON), + string(metricsJSON), + string(billingSummaryJSON), + finalChargeAmount, + currency, + string(pricingSnapshotJSON), + input.RequestFingerprint, + nullableTime(input.ResponseStartedAt), + nullableTime(input.ResponseFinishedAt), + input.ResponseDurationMS, + input.ExecutionToken, + ) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } + payloadJSON, _ := json.Marshal(map[string]any{"taskId": input.TaskID, "pricingVersion": "effective-pricing-v2"}) + _, err = tx.Exec(ctx, ` +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at +) +SELECT id, 'task.billing.settle', 'settle', $2::numeric, $3, $4::jsonb, $5::jsonb, 'pending', now() +FROM gateway_tasks +WHERE id = $1::uuid + AND run_mode = 'production' + AND gateway_user_id IS NOT NULL +ON CONFLICT (task_id, event_type) DO NOTHING`, + input.TaskID, finalChargeAmount, currency, string(pricingSnapshotJSON), string(payloadJSON)) + return err + }) + if err != nil { + return GatewayTask{}, err + } + return s.GetTask(ctx, input.TaskID) +} + +func (s *Store) FinishTaskManualReview(ctx context.Context, input FinishTaskManualReviewInput) (GatewayTask, error) { + status := strings.TrimSpace(input.TaskStatus) + if status != "succeeded" && status != "failed" { + return GatewayTask{}, fmt.Errorf("manual review task status must be succeeded or failed") + } + resultJSON, _ := json.Marshal(emptyObjectIfNil(input.Result)) + pricingSnapshotJSON, _ := json.Marshal(emptyObjectIfNil(input.PricingSnapshot)) + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + if strings.TrimSpace(input.AttemptID) != "" { + attemptStatus := "failed" + if status == "succeeded" { + attemptStatus = "succeeded" + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_task_attempts +SET status = $2, + request_id = NULLIF($3, ''), + error_code = CASE WHEN $2 = 'failed' THEN NULLIF($4, '') ELSE NULL END, + error_message = CASE WHEN $2 = 'failed' THEN NULLIF($5, '') ELSE NULL END, + upstream_submission_status = CASE WHEN $2 = 'succeeded' THEN 'response_received' ELSE upstream_submission_status END, + upstream_submission_updated_at = now(), + response_started_at = $6::timestamptz, + response_finished_at = $7::timestamptz, + response_duration_ms = $8, + finished_at = now(), + updated_at = now() +WHERE id = $1::uuid`, input.AttemptID, attemptStatus, input.RequestID, input.Code, input.Message, + nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS); err != nil { + return err + } + } + tag, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET status = $2, + result = $3::jsonb, + request_id = NULLIF($4, ''), + error = CASE WHEN $2 = 'failed' THEN NULLIF($6, '') ELSE NULL END, + error_code = NULLIF($5, ''), + error_message = NULLIF($6, ''), + billing_status = 'manual_review', + pricing_snapshot = CASE WHEN $7::jsonb = '{}'::jsonb THEN pricing_snapshot ELSE $7::jsonb END, + request_fingerprint = COALESCE(NULLIF($8, ''), request_fingerprint), + billing_updated_at = now(), + response_started_at = $9::timestamptz, + response_finished_at = $10::timestamptz, + response_duration_ms = $11, + locked_by = NULL, + locked_at = NULL, + heartbeat_at = NULL, + execution_token = NULL, + execution_lease_expires_at = NULL, + finished_at = now(), + updated_at = now() +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $12::uuid`, input.TaskID, status, string(resultJSON), input.RequestID, input.Code, input.Message, + string(pricingSnapshotJSON), input.RequestFingerprint, nullableTime(input.ResponseStartedAt), + nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS, input.ExecutionToken) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } + return nil + }) + if err != nil { return GatewayTask{}, err } return s.GetTask(ctx, input.TaskID) @@ -743,6 +982,9 @@ ON CONFLICT (gateway_user_id, currency) DO NOTHING`, task.GatewayTenantID, task.GatewayUserID, task.TenantID, task.TenantKey, task.UserID, currency); err != nil { return err } + if err := ensureWalletAccountAuditGuard(ctx, tx, task.GatewayUserID, currency); err != nil { + return err + } var exists bool var accountID string var balanceBefore float64 @@ -859,7 +1101,8 @@ func taskBillingString(value any) string { func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureInput) (GatewayTask, error) { metricsJSON, _ := json.Marshal(emptyObjectIfNil(input.Metrics)) resultJSON, _ := json.Marshal(emptyObjectIfNil(input.Result)) - if _, err := s.pool.Exec(ctx, ` + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + tag, err := tx.Exec(ctx, ` UPDATE gateway_tasks SET status = 'failed', error = NULLIF($2::text, ''), @@ -871,22 +1114,55 @@ func (s *Store) FinishTaskFailure(ctx context.Context, input FinishTaskFailureIn response_finished_at = $7::timestamptz, response_duration_ms = $8, result = $9::jsonb, + billing_status = CASE + WHEN run_mode <> 'production' OR gateway_user_id IS NULL THEN 'not_required' + WHEN reservation_amount > 0 THEN 'pending' + ELSE 'released' + END, + billing_updated_at = now(), locked_by = NULL, locked_at = NULL, heartbeat_at = NULL, + execution_token = NULL, + execution_lease_expires_at = NULL, finished_at = now(), updated_at = now() -WHERE id = $1::uuid`, - input.TaskID, - input.Message, - input.Code, - input.RequestID, - string(metricsJSON), - nullableTime(input.ResponseStartedAt), - nullableTime(input.ResponseFinishedAt), - input.ResponseDurationMS, - string(resultJSON), - ); err != nil { +WHERE id = $1::uuid + AND status = 'running' + AND execution_token = $10::uuid`, + input.TaskID, + input.Message, + input.Code, + input.RequestID, + string(metricsJSON), + nullableTime(input.ResponseStartedAt), + nullableTime(input.ResponseFinishedAt), + input.ResponseDurationMS, + string(resultJSON), + input.ExecutionToken, + ) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return ErrTaskExecutionLeaseLost + } + payloadJSON, _ := json.Marshal(map[string]any{"taskId": input.TaskID, "reason": input.Code}) + _, err = tx.Exec(ctx, ` +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, pricing_snapshot, payload, status, next_attempt_at +) +SELECT id, 'task.billing.release', 'release', reservation_amount, billing_currency, + pricing_snapshot, $2::jsonb, 'pending', now() +FROM gateway_tasks +WHERE id = $1::uuid + AND run_mode = 'production' + AND gateway_user_id IS NOT NULL + AND reservation_amount > 0 +ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON)) + return err + }) + if err != nil { return GatewayTask{}, err } return s.GetTask(ctx, input.TaskID) diff --git a/apps/api/internal/store/wallet.go b/apps/api/internal/store/wallet.go index b699412..dfc996f 100644 --- a/apps/api/internal/store/wallet.go +++ b/apps/api/internal/store/wallet.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/json" + "errors" "fmt" "strconv" "strings" @@ -82,6 +83,7 @@ type WalletBalanceAdjustmentInput struct { GatewayUserID string `json:"gatewayUserId"` Currency string `json:"currency"` Balance float64 `json:"balance"` + BalanceText string `json:"-"` Reason string `json:"reason"` IdempotencyKey string `json:"idempotencyKey"` Metadata map[string]any `json:"metadata"` @@ -91,6 +93,7 @@ type WalletRechargeInput struct { GatewayUserID string `json:"gatewayUserId"` Currency string `json:"currency"` Amount float64 `json:"amount"` + AmountText string `json:"-"` Reason string `json:"reason"` IdempotencyKey string `json:"idempotencyKey"` Metadata map[string]any `json:"metadata"` @@ -135,7 +138,7 @@ func (s *Store) WalletAvailability(ctx context.Context, user *auth.User, currenc return result, nil } -func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *auth.User, billings []any) ([]WalletBillingReservation, error) { +func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user *auth.User, billings []any, pricingSnapshots ...map[string]any) ([]WalletBillingReservation, error) { gatewayUserID := taskGatewayUserID(task, user) if gatewayUserID == "" { return nil, nil @@ -145,12 +148,21 @@ func (s *Store) ReserveTaskBilling(ctx context.Context, task GatewayTask, user * return nil, fmt.Errorf("task id is required for wallet reservation") } + pricingSnapshot := map[string]any{} + if len(pricingSnapshots) > 0 { + pricingSnapshot = emptyObjectIfNil(pricingSnapshots[0]) + } + if exactAmount := walletString(pricingSnapshot["reservationAmount"]); exactAmount != "" { + return s.reserveTaskBillingExact(ctx, task, gatewayUserID, exactAmount, pricingSnapshot) + } amounts := walletBillingAmounts(billings) if len(amounts) == 0 { return nil, nil } reservations := make([]WalletBillingReservation, 0, len(amounts)) + pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot) + requestFingerprint := walletString(pricingSnapshot["requestFingerprint"]) err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { for currency, rawAmount := range amounts { amount := roundMoney(rawAmount) @@ -257,7 +269,30 @@ VALUES ( } reservations = append(reservations, reservation) } - return nil + _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks task +SET reservation_amount = COALESCE(( + SELECT SUM(reserve.amount) + FROM gateway_wallet_transactions reserve + WHERE reserve.reference_type = 'gateway_task' + AND reserve.reference_id = task.id::text + AND reserve.transaction_type = 'reserve' + AND NOT EXISTS ( + SELECT 1 + FROM gateway_wallet_transactions release + WHERE release.account_id = reserve.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = reserve.idempotency_key || ':release' + ) + ), 0), + billing_status = 'pending', + billing_currency = 'resource', + pricing_snapshot = $2::jsonb, + request_fingerprint = NULLIF($3, ''), + billing_updated_at = now(), + updated_at = now() +WHERE task.id = $1::uuid`, taskID, string(pricingSnapshotJSON), requestFingerprint) + return err }) if err != nil { return nil, err @@ -265,6 +300,106 @@ VALUES ( return reservations, err } +func (s *Store) reserveTaskBillingExact(ctx context.Context, task GatewayTask, gatewayUserID string, amount string, pricingSnapshot map[string]any) ([]WalletBillingReservation, error) { + taskID := strings.TrimSpace(task.ID) + currency := normalizeWalletCurrency(walletString(pricingSnapshot["currency"])) + if currency != "resource" { + return nil, fmt.Errorf("unsupported billing currency %q", currency) + } + pricingSnapshotJSON, _ := json.Marshal(pricingSnapshot) + requestFingerprint := walletString(pricingSnapshot["requestFingerprint"]) + var reservations []WalletBillingReservation + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var positive bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric(38, 9) > 0`, amount).Scan(&positive); err != nil { + return fmt.Errorf("invalid exact reservation amount: %w", err) + } + if !positive { + return nil + } + account, err := s.ensureWalletAccount(ctx, tx, gatewayUserID, currency) + if err != nil { + return err + } + locked, err := lockWalletAccount(ctx, tx, account.ID) + if err != nil { + return err + } + activeKey, activeAmount, err := activeWalletReservationExact(ctx, tx, locked.ID, taskID) + if err != nil { + return err + } + if activeAmount != "" { + presentationAmount, _ := strconv.ParseFloat(activeAmount, 64) + reservations = []WalletBillingReservation{{ + TaskID: taskID, AccountID: locked.ID, GatewayUserID: gatewayUserID, + GatewayTenantID: firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID), + Currency: locked.Currency, Amount: presentationAmount, IdempotencyKey: activeKey, + }} + return nil + } + sequence, err := nextWalletReservationSequence(ctx, tx, locked.ID, taskID) + if err != nil { + return err + } + key := billingReservationIdempotencyKey(taskID, locked.Currency, sequence) + var balanceBefore string + var frozenBefore string + if err := tx.QueryRow(ctx, ` +SELECT balance::text, frozen_balance::text +FROM gateway_wallet_accounts +WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore, &frozenBefore); err != nil { + return err + } + var frozenAfter string + err = tx.QueryRow(ctx, ` +UPDATE gateway_wallet_accounts +SET frozen_balance = frozen_balance + $2::numeric(38, 9), updated_at = now() +WHERE id = $1::uuid + AND balance - frozen_balance >= $2::numeric(38, 9) +RETURNING frozen_balance::text`, locked.ID, amount).Scan(&frozenAfter) + if errors.Is(err, pgx.ErrNoRows) { + return ErrInsufficientWalletBalance + } + if err != nil { + return err + } + metadata, _ := json.Marshal(map[string]any{ + "taskId": taskID, "reserved": amount, "balance": balanceBefore, + "frozenBefore": frozenBefore, "frozenAfter": frozenAfter, + "pricingVersion": walletString(pricingSnapshot["pricingVersion"]), + }) + if _, err := tx.Exec(ctx, ` +INSERT INTO gateway_wallet_transactions ( + account_id, gateway_tenant_id, gateway_user_id, direction, transaction_type, + amount, balance_before, balance_after, idempotency_key, reference_type, reference_id, metadata +) +VALUES ( + $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'debit', 'reserve', + $4::numeric(38, 9), $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb +)`, locked.ID, firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID), gatewayUserID, + amount, balanceBefore, key, taskID, string(metadata)); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET reservation_amount = $2::numeric(38, 9), billing_status = 'pending', + billing_currency = $3, pricing_snapshot = $4::jsonb, + request_fingerprint = NULLIF($5, ''), billing_updated_at = now(), updated_at = now() +WHERE id = $1::uuid`, taskID, amount, currency, string(pricingSnapshotJSON), requestFingerprint); err != nil { + return err + } + presentationAmount, _ := strconv.ParseFloat(amount, 64) + reservations = []WalletBillingReservation{{ + TaskID: taskID, AccountID: locked.ID, GatewayUserID: gatewayUserID, + GatewayTenantID: firstNonEmpty(locked.GatewayTenantID, task.GatewayTenantID), + Currency: locked.Currency, Amount: presentationAmount, IdempotencyKey: key, + }} + return nil + }) + return reservations, err +} + func (s *Store) ReleaseTaskBillingReservations(ctx context.Context, reservations []WalletBillingReservation, reason string) error { if len(reservations) == 0 { return nil @@ -274,10 +409,14 @@ func (s *Store) ReleaseTaskBillingReservations(ctx context.Context, reservations reason = "task_not_settled" } return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + taskIDs := map[string]struct{}{} for _, reservation := range reservations { - if reservation.Amount <= 0 || strings.TrimSpace(reservation.AccountID) == "" { + if strings.TrimSpace(reservation.AccountID) == "" { continue } + if taskID := strings.TrimSpace(reservation.TaskID); taskID != "" { + taskIDs[taskID] = struct{}{} + } reserveKey := strings.TrimSpace(reservation.IdempotencyKey) if reserveKey == "" { reserveKey = billingReservationIdempotencyKey(reservation.TaskID, reservation.Currency, 1) @@ -303,39 +442,44 @@ SELECT EXISTS ( if alreadyReleased { continue } - var storedReservedAmount float64 + var storedReservedAmount string if err := tx.QueryRow(ctx, ` SELECT COALESCE(( - SELECT amount::float8 + SELECT amount::text FROM gateway_wallet_transactions WHERE account_id = $1::uuid AND idempotency_key = $2 AND transaction_type = 'reserve' LIMIT 1 -), 0)::float8`, reservation.AccountID, reserveKey).Scan(&storedReservedAmount); err != nil { +), '0')`, reservation.AccountID, reserveKey).Scan(&storedReservedAmount); err != nil { return err } - if storedReservedAmount <= 0 { + var positive bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, storedReservedAmount).Scan(&positive); err != nil { + return err + } + if !positive { continue } - amount := roundMoney(storedReservedAmount) - frozenAfter := roundMoney(locked.FrozenBalance - amount) - if frozenAfter < 0 { - frozenAfter = 0 - } - if _, err := tx.Exec(ctx, ` + var balanceBefore string + var frozenBefore string + var frozenAfter string + if err := tx.QueryRow(ctx, ` UPDATE gateway_wallet_accounts -SET frozen_balance = $2, +SET frozen_balance = frozen_balance - $2::numeric, updated_at = now() -WHERE id = $1::uuid`, locked.ID, frozenAfter); err != nil { +WHERE id = $1::uuid + AND frozen_balance >= $2::numeric +RETURNING balance::text, (frozen_balance + $2::numeric)::text, frozen_balance::text`, + locked.ID, storedReservedAmount).Scan(&balanceBefore, &frozenBefore, &frozenAfter); err != nil { return err } metadata, _ := json.Marshal(map[string]any{ "taskId": reservation.TaskID, "reason": reason, - "reserved": amount, - "frozenBefore": roundMoney(locked.FrozenBalance), + "reserved": storedReservedAmount, + "frozenBefore": frozenBefore, "frozenAfter": frozenAfter, }) if _, err := tx.Exec(ctx, ` @@ -345,15 +489,14 @@ INSERT INTO gateway_wallet_transactions ( ) VALUES ( $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'release', - $4, $5, $6, $7, 'gateway_task', $8, $9::jsonb + $4::numeric, $5::numeric, $5::numeric, $6, 'gateway_task', $7, $8::jsonb ) ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING`, locked.ID, locked.GatewayTenantID, locked.GatewayUserID, - amount, - roundMoney(locked.Balance), - roundMoney(locked.Balance), + storedReservedAmount, + balanceBefore, releaseKey, reservation.TaskID, string(metadata), @@ -361,6 +504,33 @@ ON CONFLICT (account_id, idempotency_key) WHERE idempotency_key IS NOT NULL DO N return err } } + for taskID := range taskIDs { + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks task +SET reservation_amount = COALESCE(( + SELECT SUM(reserve.amount) + FROM gateway_wallet_transactions reserve + WHERE reserve.reference_type = 'gateway_task' + AND reserve.reference_id = task.id::text + AND reserve.transaction_type = 'reserve' + AND NOT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions release + WHERE release.account_id = reserve.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = reserve.idempotency_key || ':release' + ) + ), 0), + billing_status = CASE + WHEN status IN ('failed', 'cancelled') THEN 'released' + WHEN status = 'succeeded' THEN billing_status + ELSE 'not_started' + END, + billing_updated_at = now(), + updated_at = now() +WHERE id = $1::uuid`, taskID); err != nil { + return err + } + } return nil }) } @@ -588,8 +758,20 @@ func (s *Store) SetUserWalletBalanceTx(ctx context.Context, tx Tx, input WalletB if input.GatewayUserID == "" { return WalletAdjustmentResult{}, ErrLocalUserRequired } - if input.Balance < 0 { - return WalletAdjustmentResult{}, fmt.Errorf("wallet balance cannot be negative") + targetInput := strings.TrimSpace(input.BalanceText) + if targetInput == "" { + targetInput = strconv.FormatFloat(input.Balance, 'f', 9, 64) + } + targetBalance, err := canonicalWalletAmount(ctx, tx, targetInput) + if err != nil { + return WalletAdjustmentResult{}, err + } + var nonnegative bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric >= 0`, targetBalance).Scan(&nonnegative); err != nil { + return WalletAdjustmentResult{}, err + } + if !nonnegative { + return WalletAdjustmentResult{}, ErrInvalidWalletAmount } account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) if err != nil { @@ -608,38 +790,68 @@ FOR UPDATE`, account.ID)) return WalletAdjustmentResult{}, err } before := locked - nextBalance := roundMoney(input.Balance) - delta := roundMoney(nextBalance - locked.Balance) - if delta == 0 { + var balanceBefore string + var frozenBalance string + if err := tx.QueryRow(ctx, ` +SELECT balance::text, frozen_balance::text +FROM gateway_wallet_accounts +WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore, &frozenBalance); err != nil { + return WalletAdjustmentResult{}, err + } + var unchanged bool + var belowFrozen bool + if err := tx.QueryRow(ctx, ` +SELECT $1::numeric = $2::numeric, $1::numeric < $3::numeric`, + targetBalance, balanceBefore, frozenBalance).Scan(&unchanged, &belowFrozen); err != nil { + return WalletAdjustmentResult{}, err + } + if unchanged { return WalletAdjustmentResult{}, ErrWalletBalanceUnchanged } + if belowFrozen { + return WalletAdjustmentResult{}, ErrBalanceBelowFrozen + } direction := "credit" - amount := delta - if delta < 0 { + if strings.HasPrefix(balanceBefore, "-") { + return WalletAdjustmentResult{}, ErrInvalidWalletAmount + } + var targetGreater bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric > $2::numeric`, targetBalance, balanceBefore).Scan(&targetGreater); err != nil { + return WalletAdjustmentResult{}, err + } + if !targetGreater { direction = "debit" - amount = -delta + } + var amount string + if err := tx.QueryRow(ctx, `SELECT abs($1::numeric - $2::numeric)::text`, targetBalance, balanceBefore).Scan(&amount); err != nil { + return WalletAdjustmentResult{}, err } reason := strings.TrimSpace(input.Reason) if reason == "" { reason = "后台余额调整" } - if _, err := tx.Exec(ctx, ` + tag, err := tx.Exec(ctx, ` UPDATE gateway_wallet_accounts -SET balance = $2, - total_recharged = total_recharged + CASE WHEN $3 = 'credit' THEN $4 ELSE 0 END, +SET balance = $2::numeric, + total_recharged = total_recharged + CASE WHEN $3 = 'credit' THEN $4::numeric ELSE 0 END, updated_at = now() -WHERE id = $1::uuid`, +WHERE id = $1::uuid + AND $2::numeric >= frozen_balance`, locked.ID, - nextBalance, + targetBalance, direction, amount, - ); err != nil { + ) + if err != nil { return WalletAdjustmentResult{}, err } + if tag.RowsAffected() != 1 { + return WalletAdjustmentResult{}, ErrBalanceBelowFrozen + } metadata := mergeObjects(input.Metadata, map[string]any{ "reason": reason, - "previousBalance": roundMoney(before.Balance), - "targetBalance": nextBalance, + "previousBalance": balanceBefore, + "targetBalance": targetBalance, }) metadataJSON, _ := json.Marshal(emptyObjectIfNil(metadata)) transaction, err := scanWalletTransaction(tx.QueryRow(ctx, ` @@ -649,7 +861,7 @@ INSERT INTO gateway_wallet_transactions ( ) VALUES ( $1::uuid, NULLIF($2, '')::uuid, $3::uuid, $4, 'admin_adjust', - $5, $6, $7, NULLIF($8, ''), 'gateway_user', $9, $10::jsonb + $5::numeric, $6::numeric, $7::numeric, NULLIF($8, ''), 'gateway_user', $9, $10::jsonb ) RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''), direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8, @@ -660,8 +872,8 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA locked.GatewayUserID, direction, amount, - roundMoney(before.Balance), - nextBalance, + balanceBefore, + targetBalance, strings.TrimSpace(input.IdempotencyKey), locked.GatewayUserID, string(metadataJSON), @@ -669,11 +881,10 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA if err != nil { return WalletAdjustmentResult{}, err } - locked.Balance = nextBalance - if direction == "credit" { - locked.TotalRecharged = roundMoney(locked.TotalRecharged + amount) + locked, err = s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) + if err != nil { + return WalletAdjustmentResult{}, err } - locked.UpdatedAt = time.Now() return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil } @@ -695,9 +906,20 @@ func (s *Store) RechargeUserWalletBalanceTx(ctx context.Context, tx Tx, input Wa if input.GatewayUserID == "" { return WalletAdjustmentResult{}, ErrLocalUserRequired } - amount := roundMoney(input.Amount) - if amount <= 0 { - return WalletAdjustmentResult{}, fmt.Errorf("wallet recharge amount must be positive") + amountInput := strings.TrimSpace(input.AmountText) + if amountInput == "" { + amountInput = strconv.FormatFloat(input.Amount, 'f', 9, 64) + } + amount, err := canonicalWalletAmount(ctx, tx, amountInput) + if err != nil { + return WalletAdjustmentResult{}, err + } + var positive bool + if err := tx.QueryRow(ctx, `SELECT $1::numeric > 0`, amount).Scan(&positive); err != nil { + return WalletAdjustmentResult{}, err + } + if !positive { + return WalletAdjustmentResult{}, ErrInvalidWalletAmount } account, err := s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) if err != nil { @@ -715,26 +937,30 @@ FOR UPDATE`, account.ID)) return WalletAdjustmentResult{}, err } before := locked - nextBalance := roundMoney(locked.Balance + amount) + var balanceBefore string + if err := tx.QueryRow(ctx, `SELECT balance::text FROM gateway_wallet_accounts WHERE id = $1::uuid`, locked.ID).Scan(&balanceBefore); err != nil { + return WalletAdjustmentResult{}, err + } reason := strings.TrimSpace(input.Reason) if reason == "" { reason = "后台余额充值" } - if _, err := tx.Exec(ctx, ` + var nextBalance string + if err := tx.QueryRow(ctx, ` UPDATE gateway_wallet_accounts -SET balance = $2, - total_recharged = total_recharged + $3, +SET balance = balance + $2::numeric, + total_recharged = total_recharged + $2::numeric, updated_at = now() -WHERE id = $1::uuid`, +WHERE id = $1::uuid +RETURNING balance::text`, locked.ID, - nextBalance, amount, - ); err != nil { + ).Scan(&nextBalance); err != nil { return WalletAdjustmentResult{}, err } metadata := mergeObjects(input.Metadata, map[string]any{ "reason": reason, - "previousBalance": roundMoney(before.Balance), + "previousBalance": balanceBefore, "rechargeAmount": amount, "targetBalance": nextBalance, }) @@ -746,7 +972,7 @@ INSERT INTO gateway_wallet_transactions ( ) VALUES ( $1::uuid, NULLIF($2, '')::uuid, $3::uuid, 'credit', 'recharge', - $4, $5, $6, NULLIF($7, ''), 'gateway_user', $8, $9::jsonb + $4::numeric, $5::numeric, $6::numeric, NULLIF($7, ''), 'gateway_user', $8, $9::jsonb ) RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COALESCE(gateway_user_id::text, ''), direction, transaction_type, amount::float8, balance_before::float8, balance_after::float8, @@ -756,7 +982,7 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA locked.GatewayTenantID, locked.GatewayUserID, amount, - roundMoney(before.Balance), + balanceBefore, nextBalance, strings.TrimSpace(input.IdempotencyKey), locked.GatewayUserID, @@ -765,12 +991,21 @@ RETURNING id::text, account_id::text, COALESCE(gateway_tenant_id::text, ''), COA if err != nil { return WalletAdjustmentResult{}, err } - locked.Balance = nextBalance - locked.TotalRecharged = roundMoney(locked.TotalRecharged + amount) - locked.UpdatedAt = time.Now() + locked, err = s.ensureWalletAccount(ctx, tx, input.GatewayUserID, input.Currency) + if err != nil { + return WalletAdjustmentResult{}, err + } return WalletAdjustmentResult{Account: locked, Before: before, Transaction: transaction}, nil } +func canonicalWalletAmount(ctx context.Context, q Tx, value string) (string, error) { + var canonical string + if err := q.QueryRow(ctx, `SELECT $1::numeric(38, 9)::text`, strings.TrimSpace(value)).Scan(&canonical); err != nil { + return "", fmt.Errorf("%w: %v", ErrInvalidWalletAmount, err) + } + return canonical, nil +} + func (s *Store) ensureWalletAccount(ctx context.Context, q Tx, gatewayUserID string, currency string) (GatewayWalletAccount, error) { currency = normalizeWalletCurrency(currency) if _, err := q.Exec(ctx, ` @@ -794,6 +1029,9 @@ WHERE gateway_wallet_accounts.gateway_tenant_id IS NULL OR COALESCE(gateway_wallet_accounts.user_id, '') = ''`, gatewayUserID, currency); err != nil { return GatewayWalletAccount{}, err } + if err := ensureWalletAccountAuditGuard(ctx, q, gatewayUserID, currency); err != nil { + return GatewayWalletAccount{}, err + } account, err := scanWalletAccount(q.QueryRow(ctx, ` SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text, COALESCE(tenant_id, ''), COALESCE(tenant_key, ''), COALESCE(user_id, ''), @@ -811,6 +1049,17 @@ WHERE gateway_user_id = $1::uuid return account, nil } +func ensureWalletAccountAuditGuard(ctx context.Context, q Tx, gatewayUserID string, currency string) error { + _, err := q.Exec(ctx, ` +INSERT INTO gateway_wallet_account_audit_guards (account_id) +SELECT id +FROM gateway_wallet_accounts +WHERE gateway_user_id = $1::uuid + AND currency = $2 +ON CONFLICT (account_id) DO NOTHING`, gatewayUserID, normalizeWalletCurrency(currency)) + return err +} + func lockWalletAccount(ctx context.Context, tx pgx.Tx, accountID string) (GatewayWalletAccount, error) { return scanWalletAccount(tx.QueryRow(ctx, ` SELECT id::text, COALESCE(gateway_tenant_id::text, ''), gateway_user_id::text, @@ -851,6 +1100,31 @@ LIMIT 1`, accountID, taskID).Scan(&key, &amount) return key, roundMoney(amount), nil } +func activeWalletReservationExact(ctx context.Context, tx pgx.Tx, accountID string, taskID string) (string, string, error) { + var key string + var amount string + err := tx.QueryRow(ctx, ` +SELECT COALESCE(t.idempotency_key, ''), t.amount::text +FROM gateway_wallet_transactions t +WHERE t.account_id = $1::uuid + AND t.reference_type = 'gateway_task' + AND t.reference_id = $2 + AND t.transaction_type = 'reserve' + AND COALESCE(t.idempotency_key, '') <> '' + AND NOT EXISTS ( + SELECT 1 FROM gateway_wallet_transactions release + WHERE release.account_id = t.account_id + AND release.transaction_type = 'release' + AND release.idempotency_key = t.idempotency_key || ':release' + ) +ORDER BY t.created_at DESC +LIMIT 1`, accountID, taskID).Scan(&key, &amount) + if errors.Is(err, pgx.ErrNoRows) { + return "", "", nil + } + return key, amount, err +} + func nextWalletReservationSequence(ctx context.Context, tx pgx.Tx, accountID string, taskID string) (int, error) { var count int if err := tx.QueryRow(ctx, ` diff --git a/apps/api/internal/store/wallet_reservation_test.go b/apps/api/internal/store/wallet_reservation_test.go index 79497ef..b9ec265 100644 --- a/apps/api/internal/store/wallet_reservation_test.go +++ b/apps/api/internal/store/wallet_reservation_test.go @@ -132,6 +132,10 @@ RETURNING id::text`, "wallet-reservation-user-"+suffix, "wallet_reservation_"+su } t.Cleanup(func() { cleanupCtx := context.Background() + _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_wallet_transactions WHERE gateway_user_id = $1::uuid`, userID) + _, _ = db.pool.Exec(cleanupCtx, ` +DELETE FROM gateway_wallet_account_audit_guards +WHERE account_id IN (SELECT id FROM gateway_wallet_accounts WHERE gateway_user_id = $1::uuid)`, userID) _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_users WHERE id = $1::uuid`, userID) _, _ = db.pool.Exec(cleanupCtx, `DELETE FROM gateway_tenants WHERE id = $1::uuid`, tenantID) }) diff --git a/apps/api/migrations/0069_billing_correctness_v2.sql b/apps/api/migrations/0069_billing_correctness_v2.sql index 87b0f9e..3ababdb 100644 --- a/apps/api/migrations/0069_billing_correctness_v2.sql +++ b/apps/api/migrations/0069_billing_correctness_v2.sql @@ -1,16 +1,24 @@ ALTER TABLE gateway_tasks - ADD COLUMN IF NOT EXISTS billing_version text NOT NULL DEFAULT 'effective-pricing-v2', - ADD COLUMN IF NOT EXISTS billing_status text NOT NULL DEFAULT 'not_started', - ADD COLUMN IF NOT EXISTS billing_currency text NOT NULL DEFAULT 'resource', - ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS billing_version text DEFAULT 'effective-pricing-v2', + ADD COLUMN IF NOT EXISTS billing_status text DEFAULT 'not_started', + ADD COLUMN IF NOT EXISTS billing_currency text DEFAULT 'resource', + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb, ADD COLUMN IF NOT EXISTS request_fingerprint text, ADD COLUMN IF NOT EXISTS idempotency_key_hash text, - ADD COLUMN IF NOT EXISTS reservation_amount numeric(38, 9) NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS idempotency_request_hash text, + ADD COLUMN IF NOT EXISTS reservation_amount numeric(38, 9) DEFAULT 0, ADD COLUMN IF NOT EXISTS execution_token uuid, ADD COLUMN IF NOT EXISTS execution_lease_expires_at timestamptz, ADD COLUMN IF NOT EXISTS billing_updated_at timestamptz, ADD COLUMN IF NOT EXISTS billing_settled_at timestamptz; +UPDATE gateway_tasks +SET billing_version = COALESCE(billing_version, 'effective-pricing-v2'), + billing_status = COALESCE(billing_status, 'not_started'), + billing_currency = COALESCE(billing_currency, 'resource'), + pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb), + reservation_amount = COALESCE(reservation_amount, 0); + ALTER TABLE gateway_tasks ADD CONSTRAINT gateway_tasks_billing_status_v2_check CHECK ( billing_status IN ( @@ -32,28 +40,40 @@ CREATE INDEX IF NOT EXISTS idx_gateway_tasks_billing_status_v2 ON gateway_tasks(billing_status, billing_updated_at, created_at); ALTER TABLE gateway_task_attempts - ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb, ADD COLUMN IF NOT EXISTS request_fingerprint text, - ADD COLUMN IF NOT EXISTS upstream_submission_status text NOT NULL DEFAULT 'not_submitted', + ADD COLUMN IF NOT EXISTS upstream_submission_status text DEFAULT 'not_submitted', ADD COLUMN IF NOT EXISTS upstream_submission_updated_at timestamptz; +UPDATE gateway_task_attempts +SET pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb), + upstream_submission_status = COALESCE(upstream_submission_status, 'not_submitted'); + ALTER TABLE gateway_task_attempts ADD CONSTRAINT gateway_task_attempts_submission_v2_check CHECK ( upstream_submission_status IN ('not_submitted', 'submitting', 'response_received') ) NOT VALID; ALTER TABLE settlement_outbox - ADD COLUMN IF NOT EXISTS action text NOT NULL DEFAULT 'settle', - ADD COLUMN IF NOT EXISTS amount numeric(38, 9) NOT NULL DEFAULT 0, - ADD COLUMN IF NOT EXISTS currency text NOT NULL DEFAULT 'resource', - ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN IF NOT EXISTS action text DEFAULT 'settle', + ADD COLUMN IF NOT EXISTS amount numeric(38, 9) DEFAULT 0, + ADD COLUMN IF NOT EXISTS currency text DEFAULT 'resource', + ADD COLUMN IF NOT EXISTS pricing_snapshot jsonb DEFAULT '{}'::jsonb, ADD COLUMN IF NOT EXISTS locked_by text, ADD COLUMN IF NOT EXISTS lock_token uuid, ADD COLUMN IF NOT EXISTS locked_at timestamptz, ADD COLUMN IF NOT EXISTS last_error_code text, ADD COLUMN IF NOT EXISTS last_error_message text, ADD COLUMN IF NOT EXISTS completed_at timestamptz, - ADD COLUMN IF NOT EXISTS manual_review_reason text; + ADD COLUMN IF NOT EXISTS manual_review_reason text, + ADD COLUMN IF NOT EXISTS retry_idempotency_key_hash text, + ADD COLUMN IF NOT EXISTS retry_requested_at timestamptz; + +UPDATE settlement_outbox +SET action = COALESCE(action, 'settle'), + amount = COALESCE(amount, 0), + currency = COALESCE(currency, 'resource'), + pricing_snapshot = COALESCE(pricing_snapshot, '{}'::jsonb); ALTER TABLE settlement_outbox ADD CONSTRAINT settlement_outbox_action_v2_check CHECK (action IN ('settle', 'release')) NOT VALID, @@ -71,7 +91,9 @@ CREATE INDEX IF NOT EXISTS idx_settlement_outbox_stale_v2 WHERE status = 'processing'; ALTER TABLE model_pricing_rules - ADD COLUMN IF NOT EXISTS is_free boolean NOT NULL DEFAULT false; + ADD COLUMN IF NOT EXISTS is_free boolean DEFAULT false; + +UPDATE model_pricing_rules SET is_free = false WHERE is_free IS NULL; ALTER TABLE model_pricing_rules ADD CONSTRAINT model_pricing_rules_explicit_free_v2_check CHECK ( @@ -82,9 +104,18 @@ ALTER TABLE gateway_wallet_accounts ADD CONSTRAINT gateway_wallet_nonnegative_frozen_v2_check CHECK (frozen_balance >= 0) NOT VALID, ADD CONSTRAINT gateway_wallet_balance_covers_frozen_v2_check CHECK (balance >= frozen_balance) NOT VALID; +CREATE TABLE IF NOT EXISTS gateway_wallet_account_audit_guards ( + account_id uuid PRIMARY KEY REFERENCES gateway_wallet_accounts(id) ON DELETE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO gateway_wallet_account_audit_guards (account_id) +SELECT id FROM gateway_wallet_accounts +ON CONFLICT (account_id) DO NOTHING; + ALTER TABLE gateway_wallet_transactions - ADD CONSTRAINT gateway_wallet_transactions_account_restrict_v2 - FOREIGN KEY (account_id) REFERENCES gateway_wallet_accounts(id) ON DELETE RESTRICT NOT VALID; + ADD CONSTRAINT gateway_wallet_transactions_audit_guard_v2 + FOREIGN KEY (account_id) REFERENCES gateway_wallet_account_audit_guards(account_id) ON DELETE RESTRICT NOT VALID; UPDATE gateway_tasks task SET billing_status = 'not_required', -- 2.54.0 From 62d25fcb111616cf6e10c71a12250a83bf025e4e Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 00:25:57 +0800 Subject: [PATCH 05/13] =?UTF-8?q?feat(web):=20=E5=A2=9E=E5=8A=A0=E8=AE=A1?= =?UTF-8?q?=E8=B4=B9=E7=BB=93=E7=AE=97=E5=BC=82=E5=B8=B8=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E7=95=8C=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 管理端支持按状态查看结算与释放记录，展示精确到九位的小数金额、错误分类和重试次数，并使用一次性 Idempotency-Key 发起审计化重试。 --- apps/web/src/api.ts | 24 ++++ apps/web/src/pages/AdminPage.tsx | 22 +-- .../pages/admin/BillingSettlementsPanel.tsx | 131 ++++++++++++++++++ packages/contracts/src/index.ts | 46 ++++++ 4 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 apps/web/src/pages/admin/BillingSettlementsPanel.tsx diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index f367ae3..834b372 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -3,6 +3,8 @@ import type { AuthUser, BaseModelCatalogItem, BaseModelUpsertRequest, + BillingSettlementListResponse, + BillingSettlementRetryResponse, CatalogProvider, CatalogProviderUpsertRequest, ClientCustomizationSettings, @@ -1000,6 +1002,28 @@ export async function restoreModelRuntimeStatus(token: string, platformModelId: }); } +export async function listBillingSettlements( + token: string, + input: { status?: string; action?: string; page?: number; pageSize?: number; signal?: AbortSignal } = {}, +): Promise { + const search = new URLSearchParams({ + page: String(input.page ?? 1), + pageSize: String(input.pageSize ?? 50), + }); + if (input.status) search.set('status', input.status); + if (input.action) search.set('action', input.action); + return request(`/api/admin/runtime/billing-settlements?${search.toString()}`, { token, signal: input.signal }); +} + +export async function retryBillingSettlement(token: string, settlementId: string): Promise { + const idempotencyKey = crypto.randomUUID(); + return request(`/api/admin/runtime/billing-settlements/${settlementId}/retry`, { + method: 'POST', + token, + headers: { 'Idempotency-Key': idempotencyKey }, + }); +} + export async function getNetworkProxyConfig(token: string): Promise { return request('/api/admin/config/network-proxy', { token }); } diff --git a/apps/web/src/pages/AdminPage.tsx b/apps/web/src/pages/AdminPage.tsx index 59404bd..f3649fc 100644 --- a/apps/web/src/pages/AdminPage.tsx +++ b/apps/web/src/pages/AdminPage.tsx @@ -26,6 +26,7 @@ import type { AdminSection, LoadState, PlatformWithModelsInput } from '../types' import { AccessRulesPanel } from './admin/AccessRulesPanel'; import { AuditLogsPanel } from './admin/AuditLogsPanel'; import { BaseModelCatalogPanel } from './admin/BaseModelCatalogPanel'; +import { BillingSettlementsPanel } from './admin/BillingSettlementsPanel'; import { TenantsPanel, UserGroupsPanel, UsersPanel } from './admin/IdentityManagementPanels'; import { PlatformManagementPanel } from './admin/PlatformManagementPanel'; import { PricingRulesPanel } from './admin/PricingRulesPanel'; @@ -131,15 +132,18 @@ export function AdminPage(props: { /> )} {props.section === 'runtime' && ( - +
+ + +
)} {props.section === 'accessRules' && ( ([]); + const [status, setStatus] = useState(''); + const [loading, setLoading] = useState(false); + const [retrying, setRetrying] = useState(''); + const [message, setMessage] = useState(''); + + const load = useCallback(async (signal?: AbortSignal) => { + setLoading(true); + setMessage(''); + try { + const result = await listBillingSettlements(props.token, { status, pageSize: 100, signal }); + setItems(result.items); + } catch (error) { + if (signal?.aborted) return; + setMessage(error instanceof Error ? error.message : '结算队列加载失败'); + } finally { + if (!signal?.aborted) setLoading(false); + } + }, [props.token, status]); + + useEffect(() => { + const controller = new AbortController(); + void load(controller.signal); + return () => controller.abort(); + }, [load]); + + async function retry(item: BillingSettlement) { + setRetrying(item.id); + setMessage(''); + try { + await retryBillingSettlement(props.token, item.id); + setMessage(`结算 ${shortId(item.id)} 已重新进入队列`); + await load(); + } catch (error) { + setMessage(error instanceof Error ? error.message : '结算重试失败'); + } finally { + setRetrying(''); + } + } + + const exceptions = items.filter((item) => item.status === 'retryable_failed' || item.status === 'manual_review').length; + return ( +
+ + +
+
+
+ 计费结算 +

任务结果与账务状态相互独立；异常记录可审计重试，不会重新调用上游。

+
+ {exceptions ? `${exceptions} 条异常` : `${items.length} 条`} +
+ + +
+ + +
+ {message &&

{message}

} + + + {items.length ? ( + + + 任务 / 动作 + 状态 + 金额 + 尝试 + 异常分类 + 操作 + + {items.map((item) => ( + + {shortId(item.taskId)}{item.action === 'settle' ? '扣费' : '释放冻结'} · {shortId(item.id)} + {statusLabel(item.status)} + {formatAmount(item.amount)} {item.currency} + {item.attempts} / 20 + {item.lastErrorCode || item.manualReviewReason || '-'} + + {(item.status === 'retryable_failed' || item.status === 'manual_review') ? ( + + ) : '-'} + + + ))} +
+ ) : ( + {loading ? '正在加载结算队列' : '暂无结算记录'}可切换状态筛选，查看待处理与人工复核记录。 + )} +
+ ); +} + +function statusLabel(status: string) { + return ({ pending: '待处理', processing: '处理中', retryable_failed: '等待重试', manual_review: '人工复核', completed: '已完成' } as Record)[status] ?? status; +} + +function statusVariant(status: string): 'secondary' | 'warning' | 'destructive' | 'success' | 'outline' { + if (status === 'completed') return 'success'; + if (status === 'manual_review') return 'destructive'; + if (status === 'retryable_failed') return 'warning'; + if (status === 'processing') return 'outline'; + return 'secondary'; +} + +function formatAmount(value: number) { + return new Intl.NumberFormat('zh-CN', { minimumFractionDigits: 0, maximumFractionDigits: 9 }).format(value); +} + +function shortId(value: string) { + return value.length > 12 ? `${value.slice(0, 8)}…${value.slice(-4)}` : value; +} diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index ea9504d..4f19f2e 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1156,6 +1156,15 @@ export interface GatewayTask { metrics?: Record; billingSummary?: Record; finalChargeAmount?: number; + billingVersion?: string; + billingStatus?: 'not_started' | 'pending' | 'processing' | 'settled' | 'released' | 'retryable_failed' | 'manual_review' | 'not_required' | string; + billingCurrency?: string; + pricingSnapshot?: Record; + requestFingerprint?: string; + reservationAmount?: number; + executionLeaseExpiresAt?: string; + billingUpdatedAt?: string; + billingSettledAt?: string; responseStartedAt?: string; responseFinishedAt?: string; responseDurationMs?: number; @@ -1212,12 +1221,49 @@ export interface GatewayTaskAttempt { responseStartedAt?: string; responseFinishedAt?: string; responseDurationMs?: number; + pricingSnapshot?: Record; + requestFingerprint?: string; + upstreamSubmissionStatus?: 'not_submitted' | 'submitting' | 'response_received' | string; + upstreamSubmissionUpdatedAt?: string; errorCode?: string; errorMessage?: string; startedAt: string; finishedAt?: string; } +export interface BillingSettlement { + id: string; + taskId: string; + action: 'settle' | 'release' | string; + amount: number; + currency: string; + status: 'pending' | 'processing' | 'retryable_failed' | 'completed' | 'manual_review' | string; + attempts: number; + nextAttemptAt: string; + lockedBy?: string; + lockedAt?: string; + lastErrorCode?: string; + lastErrorMessage?: string; + manualReviewReason?: string; + pricingSnapshot?: Record; + payload?: Record; + completedAt?: string; + createdAt: string; + updatedAt: string; +} + +export interface BillingSettlementListResponse { + items: BillingSettlement[]; + total: number; + page: number; + pageSize: number; +} + +export interface BillingSettlementRetryResponse { + settlement: BillingSettlement; + auditLog?: GatewayAuditLog; +} + export interface GatewayTaskEvent { id: string; taskId: string; -- 2.54.0 From c879de18e2ecec6ffb3646b0dac758148ab08894 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 00:26:11 +0800 Subject: [PATCH 06/13] =?UTF-8?q?docs(api):=20=E5=90=8C=E6=AD=A5=E8=AE=A1?= =?UTF-8?q?=E8=B4=B9=E7=BB=93=E7=AE=97=E6=8E=A5=E5=8F=A3=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重新生成 OpenAPI，补充估价字段、任务计费状态、管理员结算查询与审计重试接口。 --- apps/api/docs/swagger.json | 440 +++++++++++++++++++++++++++++++++++++ apps/api/docs/swagger.yaml | 292 ++++++++++++++++++++++++ 2 files changed, 732 insertions(+) diff --git a/apps/api/docs/swagger.json b/apps/api/docs/swagger.json index a4ea972..4a77aad 100644 --- a/apps/api/docs/swagger.json +++ b/apps/api/docs/swagger.json @@ -1963,6 +1963,160 @@ } } }, + "/api/admin/runtime/billing-settlements": { + "get": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "管理端分页查询待结算、重试失败和人工复核记录。", + "produces": [ + "application/json" + ], + "tags": [ + "billing" + ], + "summary": "查询计费结算队列", + "parameters": [ + { + "type": "string", + "description": "结算状态", + "name": "status", + "in": "query" + }, + { + "type": "string", + "description": "动作：settle 或 release", + "name": "action", + "in": "query" + }, + { + "type": "integer", + "default": 1, + "description": "页码", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "default": 50, + "description": "每页数量", + "name": "pageSize", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/httpapi.BillingSettlementListResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + } + } + } + }, + "/api/admin/runtime/billing-settlements/{settlementId}/retry": { + "post": { + "security": [ + { + "BearerAuth": [] + } + ], + "description": "Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。", + "produces": [ + "application/json" + ], + "tags": [ + "billing" + ], + "summary": "重试计费结算", + "parameters": [ + { + "type": "string", + "description": "结算记录 ID", + "name": "settlementId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "幂等键", + "name": "Idempotency-Key", + "in": "header", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/httpapi.BillingSettlementRetryResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/httpapi.ErrorEnvelope" + } + } + } + } + }, "/api/admin/runtime/model-rate-limits": { "get": { "security": [ @@ -5340,6 +5494,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -5501,6 +5661,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -5594,6 +5760,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -5804,6 +5976,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -6202,6 +6380,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -6445,6 +6629,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -6538,6 +6728,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -6940,6 +7136,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -7033,6 +7235,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -7853,6 +8061,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -7966,6 +8180,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -8059,6 +8279,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -8152,6 +8378,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -8271,6 +8503,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -8440,6 +8678,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -8533,6 +8777,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9118,6 +9368,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9279,6 +9535,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9372,6 +9634,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9465,6 +9733,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9558,6 +9832,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9727,6 +10007,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9820,6 +10106,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -9971,6 +10263,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -10167,6 +10465,12 @@ "name": "X-Async", "in": "header" }, + { + "type": "string", + "description": "可选请求幂等键；同一用户范围内唯一", + "name": "Idempotency-Key", + "in": "header" + }, { "description": "AI 任务请求，字段随任务类型变化", "name": "input", @@ -10469,6 +10773,40 @@ } } }, + "httpapi.BillingSettlementListResponse": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/store.BillingSettlement" + } + }, + "page": { + "type": "integer", + "example": 1 + }, + "pageSize": { + "type": "integer", + "example": 50 + }, + "total": { + "type": "integer", + "example": 42 + } + } + }, + "httpapi.BillingSettlementRetryResponse": { + "type": "object", + "properties": { + "auditLog": { + "$ref": "#/definitions/store.AuditLog" + }, + "settlement": { + "$ref": "#/definitions/store.BillingSettlement" + } + } + }, "httpapi.CatalogProviderListResponse": { "type": "object", "properties": { @@ -12658,6 +12996,67 @@ } } }, + "store.BillingSettlement": { + "type": "object", + "properties": { + "action": { + "type": "string" + }, + "amount": { + "type": "number" + }, + "attempts": { + "type": "integer" + }, + "completedAt": { + "type": "string" + }, + "createdAt": { + "type": "string" + }, + "currency": { + "type": "string" + }, + "id": { + "type": "string" + }, + "lastErrorCode": { + "type": "string" + }, + "lastErrorMessage": { + "type": "string" + }, + "lockedAt": { + "type": "string" + }, + "lockedBy": { + "type": "string" + }, + "manualReviewReason": { + "type": "string" + }, + "nextAttemptAt": { + "type": "string" + }, + "payload": { + "type": "object", + "additionalProperties": {} + }, + "pricingSnapshot": { + "type": "object", + "additionalProperties": {} + }, + "status": { + "type": "string" + }, + "taskId": { + "type": "string" + }, + "updatedAt": { + "type": "string" + } + } + }, "store.CatalogProvider": { "type": "object", "properties": { @@ -13089,10 +13488,25 @@ "$ref": "#/definitions/store.TaskAttempt" } }, + "billingCurrency": { + "type": "string" + }, + "billingSettledAt": { + "type": "string" + }, + "billingStatus": { + "type": "string" + }, "billingSummary": { "type": "object", "additionalProperties": {} }, + "billingUpdatedAt": { + "type": "string" + }, + "billingVersion": { + "type": "string" + }, "billings": { "type": "array", "items": {} @@ -13115,6 +13529,9 @@ "errorMessage": { "type": "string" }, + "executionLeaseExpiresAt": { + "type": "string" + }, "finalChargeAmount": { "type": "number" }, @@ -13149,6 +13566,10 @@ "newMessageCount": { "type": "integer" }, + "pricingSnapshot": { + "type": "object", + "additionalProperties": {} + }, "remoteTaskId": { "type": "string" }, @@ -13160,12 +13581,18 @@ "type": "object", "additionalProperties": {} }, + "requestFingerprint": { + "type": "string" + }, "requestId": { "type": "string" }, "requestedModel": { "type": "string" }, + "reservationAmount": { + "type": "number" + }, "resolvedModel": { "type": "string" }, @@ -14506,6 +14933,10 @@ "platformName": { "type": "string" }, + "pricingSnapshot": { + "type": "object", + "additionalProperties": {} + }, "provider": { "type": "string" }, @@ -14515,6 +14946,9 @@ "queueKey": { "type": "string" }, + "requestFingerprint": { + "type": "string" + }, "requestId": { "type": "string" }, @@ -14553,6 +14987,12 @@ "taskId": { "type": "string" }, + "upstreamSubmissionStatus": { + "type": "string" + }, + "upstreamSubmissionUpdatedAt": { + "type": "string" + }, "usage": { "type": "object", "additionalProperties": {} diff --git a/apps/api/docs/swagger.yaml b/apps/api/docs/swagger.yaml index c6fc9fc..cb7e255 100644 --- a/apps/api/docs/swagger.yaml +++ b/apps/api/docs/swagger.yaml @@ -85,6 +85,29 @@ definitions: $ref: '#/definitions/store.BaseModel' type: array type: object + httpapi.BillingSettlementListResponse: + properties: + items: + items: + $ref: '#/definitions/store.BillingSettlement' + type: array + page: + example: 1 + type: integer + pageSize: + example: 50 + type: integer + total: + example: 42 + type: integer + type: object + httpapi.BillingSettlementRetryResponse: + properties: + auditLog: + $ref: '#/definitions/store.AuditLog' + settlement: + $ref: '#/definitions/store.BillingSettlement' + type: object httpapi.CatalogProviderListResponse: properties: items: @@ -1599,6 +1622,47 @@ definitions: status: type: string type: object + store.BillingSettlement: + properties: + action: + type: string + amount: + type: number + attempts: + type: integer + completedAt: + type: string + createdAt: + type: string + currency: + type: string + id: + type: string + lastErrorCode: + type: string + lastErrorMessage: + type: string + lockedAt: + type: string + lockedBy: + type: string + manualReviewReason: + type: string + nextAttemptAt: + type: string + payload: + additionalProperties: {} + type: object + pricingSnapshot: + additionalProperties: {} + type: object + status: + type: string + taskId: + type: string + updatedAt: + type: string + type: object store.CatalogProvider: properties: capabilitySchema: @@ -1890,9 +1954,19 @@ definitions: items: $ref: '#/definitions/store.TaskAttempt' type: array + billingCurrency: + type: string + billingSettledAt: + type: string + billingStatus: + type: string billingSummary: additionalProperties: {} type: object + billingUpdatedAt: + type: string + billingVersion: + type: string billings: items: {} type: array @@ -1908,6 +1982,8 @@ definitions: type: string errorMessage: type: string + executionLeaseExpiresAt: + type: string finalChargeAmount: type: number finishedAt: @@ -1931,6 +2007,9 @@ definitions: type: string newMessageCount: type: integer + pricingSnapshot: + additionalProperties: {} + type: object remoteTaskId: type: string remoteTaskPayload: @@ -1939,10 +2018,14 @@ definitions: request: additionalProperties: {} type: object + requestFingerprint: + type: string requestId: type: string requestedModel: type: string + reservationAmount: + type: number resolvedModel: type: string responseDurationMs: @@ -2850,12 +2933,17 @@ definitions: type: string platformName: type: string + pricingSnapshot: + additionalProperties: {} + type: object provider: type: string providerModelName: type: string queueKey: type: string + requestFingerprint: + type: string requestId: type: string requestSnapshot: @@ -2882,6 +2970,10 @@ definitions: type: integer taskId: type: string + upstreamSubmissionStatus: + type: string + upstreamSubmissionUpdatedAt: + type: string usage: additionalProperties: {} type: object @@ -4264,6 +4356,106 @@ paths: summary: 列出定价规则 tags: - pricing + /api/admin/runtime/billing-settlements: + get: + description: 管理端分页查询待结算、重试失败和人工复核记录。 + parameters: + - description: 结算状态 + in: query + name: status + type: string + - description: 动作：settle 或 release + in: query + name: action + type: string + - default: 1 + description: 页码 + in: query + name: page + type: integer + - default: 50 + description: 每页数量 + in: query + name: pageSize + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/httpapi.BillingSettlementListResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + security: + - BearerAuth: [] + summary: 查询计费结算队列 + tags: + - billing + /api/admin/runtime/billing-settlements/{settlementId}/retry: + post: + description: Manager 使用单值 Idempotency-Key 将重试失败或人工复核记录重新放入队列，并记录审计日志。 + parameters: + - description: 结算记录 ID + in: path + name: settlementId + required: true + type: string + - description: 幂等键 + in: header + name: Idempotency-Key + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/httpapi.BillingSettlementRetryResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "403": + description: Forbidden + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "404": + description: Not Found + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "409": + description: Conflict + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/httpapi.ErrorEnvelope' + security: + - BearerAuth: [] + summary: 重试计费结算 + tags: + - billing /api/admin/runtime/model-rate-limits: get: description: 管理端查看平台模型维度的限流和冷却状态。 @@ -6432,6 +6624,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -6537,6 +6733,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -6598,6 +6798,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -6732,6 +6936,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -6989,6 +7197,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -7153,6 +7365,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -7214,6 +7430,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -7473,6 +7693,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -7534,6 +7758,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8062,6 +8290,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8136,6 +8368,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8197,6 +8433,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8258,6 +8498,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8336,6 +8580,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8449,6 +8697,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8510,6 +8762,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8890,6 +9146,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -8995,6 +9255,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9056,6 +9320,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9117,6 +9385,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9178,6 +9450,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9291,6 +9567,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9352,6 +9632,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9450,6 +9734,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input @@ -9577,6 +9865,10 @@ paths: in: header name: X-Async type: boolean + - description: 可选请求幂等键；同一用户范围内唯一 + in: header + name: Idempotency-Key + type: string - description: AI 任务请求，字段随任务类型变化 in: body name: input -- 2.54.0 From 3c82c7b492af95084afbca0cbc560ad77a8be195 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 00:26:16 +0800 Subject: [PATCH 07/13] =?UTF-8?q?ci(billing):=20=E5=90=AF=E7=94=A8=20Postg?= =?UTF-8?q?reSQL=20=E8=AE=A1=E8=B4=B9=E9=9B=86=E6=88=90=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI verify 启动 PostgreSQL 16、应用全部迁移并注入 AI_GATEWAY_TEST_DATABASE_URL，使钱包、Outbox、幂等和租约集成测试不再因缺少数据库而跳过。 --- .gitea/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++ .gitea/workflows/release-ci.yml | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8adffe7..5978943 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -79,6 +79,37 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } + - name: Start PostgreSQL 16 integration database + env: + CI_RUN_ID: ${{ github.run_id }} + CI_RUN_NUMBER: ${{ github.run_number }} + run: | + set -eu + container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" + docker run -d --rm --name "$container" \ + -e POSTGRES_USER=easyai_test \ + -e POSTGRES_PASSWORD=easyai_test_only \ + -e POSTGRES_DB=easyai_gateway_test \ + postgres:16-alpine + for attempt in $(seq 1 60); do + if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then + break + fi + test "$attempt" -lt 60 + sleep 1 + done + database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") + test -n "$database_ip" + printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" + printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" + docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ + -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' + for migration in apps/api/migrations/*.sql; do + docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" + version=$(basename "$migration" .sql) + docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ + -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" + done - name: Verify Go code working-directory: apps/api env: @@ -110,3 +141,9 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . + - name: Stop PostgreSQL integration database + if: always() + run: | + if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then + docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true + fi diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml index a0d1ed5..80b389b 100644 --- a/.gitea/workflows/release-ci.yml +++ b/.gitea/workflows/release-ci.yml @@ -64,6 +64,37 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } + - name: Start PostgreSQL 16 integration database + env: + CI_RUN_ID: ${{ github.run_id }} + CI_RUN_NUMBER: ${{ github.run_number }} + run: | + set -eu + container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" + docker run -d --rm --name "$container" \ + -e POSTGRES_USER=easyai_test \ + -e POSTGRES_PASSWORD=easyai_test_only \ + -e POSTGRES_DB=easyai_gateway_test \ + postgres:16-alpine + for attempt in $(seq 1 60); do + if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then + break + fi + test "$attempt" -lt 60 + sleep 1 + done + database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") + test -n "$database_ip" + printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" + printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" + docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ + -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' + for migration in apps/api/migrations/*.sql; do + docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" + version=$(basename "$migration" .sql) + docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ + -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" + done - name: Verify Go code working-directory: apps/api env: @@ -95,3 +126,9 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . + - name: Stop PostgreSQL integration database + if: always() + run: | + if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then + docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true + fi -- 2.54.0 From 136297022940b814a950777ca30dcc241e75f065 Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 00:29:31 +0800 Subject: [PATCH 08/13] =?UTF-8?q?test(billing):=20=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E4=BC=B0=E4=BB=B7=E5=8F=AA=E8=AF=BB=E4=B8=8E=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E9=87=8D=E6=94=BE=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgreSQL HTTP 集成测试验证估价新增字段且不创建任务、冻结余额或写流水，并验证非流式重放只执行一次、异请求冲突及流式重复冲突。 --- .../httpapi/core_flow_integration_test.go | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/apps/api/internal/httpapi/core_flow_integration_test.go b/apps/api/internal/httpapi/core_flow_integration_test.go index a400451..351db89 100644 --- a/apps/api/internal/httpapi/core_flow_integration_test.go +++ b/apps/api/internal/httpapi/core_flow_integration_test.go @@ -48,6 +48,7 @@ func TestCoreLocalFlow(t *testing.T) { DatabaseURL: databaseURL, IdentityMode: "hybrid", JWTSecret: "test-secret", + BillingEngineMode: "enforce", TaskProgressCallbackEnabled: true, TaskProgressCallbackURL: "http://callback.local/task-progress", CORSAllowedOrigin: "*", @@ -814,6 +815,7 @@ LIMIT 1`).Scan(&gptImageModelTypesRaw); err != nil { "currency": "resource", "rules": []map[string]any{ {"ruleKey": "text_input", "displayName": "Text Input", "resourceType": "text_input", "unit": "1k_tokens", "basePrice": 1}, + {"ruleKey": "text_cached_input", "displayName": "Text Cached Input", "resourceType": "text_cached_input", "unit": "1k_tokens", "basePrice": 0.5}, {"ruleKey": "text_output", "displayName": "Text Output", "resourceType": "text_output", "unit": "1k_tokens", "basePrice": 2}, {"ruleKey": "image", "displayName": "Image", "resourceType": "image", "unit": "image", "basePrice": 7}, {"ruleKey": "image_edit", "displayName": "Image Edit", "resourceType": "image_edit", "unit": "image", "basePrice": 11}, @@ -845,6 +847,101 @@ WHERE gateway_user_id = $1::uuid AND currency = 'resource'`, smokeGatewayUserID).Scan(&walletBalanceBefore); err != nil { t.Fatalf("read wallet balance before pricing task: %v", err) } + var estimateSideEffectsBefore struct { + Tasks int + Transactions int + Frozen string + } + if err := testPool.QueryRow(ctx, ` +SELECT (SELECT count(*) FROM gateway_tasks), + (SELECT count(*) FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid), + (SELECT frozen_balance::text FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid AND currency='resource')`, + smokeGatewayUserID, + ).Scan(&estimateSideEffectsBefore.Tasks, &estimateSideEffectsBefore.Transactions, &estimateSideEffectsBefore.Frozen); err != nil { + t.Fatalf("read estimate side effects before request: %v", err) + } + var pricingEstimate struct { + TotalAmount float64 `json:"totalAmount"` + ReservationAmount float64 `json:"reservationAmount"` + CandidateCount int `json:"candidateCount"` + PricingVersion string `json:"pricingVersion"` + RequestFingerprint string `json:"requestFingerprint"` + } + doJSON(t, server.URL, http.MethodPost, "/api/v1/pricing/estimate", apiKeyResponse.Secret, map[string]any{ + "kind": "chat.completions", "model": pricingModel, "max_completion_tokens": 32, + "messages": []map[string]any{{"role": "user", "content": "estimate only"}}, + }, http.StatusOK, &pricingEstimate) + if pricingEstimate.TotalAmount <= 0 || pricingEstimate.ReservationAmount < pricingEstimate.TotalAmount || + pricingEstimate.CandidateCount != 1 || pricingEstimate.PricingVersion != "effective-pricing-v2" || pricingEstimate.RequestFingerprint == "" { + t.Fatalf("unexpected effective pricing estimate: %+v", pricingEstimate) + } + var estimateSideEffectsAfter struct { + Tasks int + Transactions int + Frozen string + } + if err := testPool.QueryRow(ctx, ` +SELECT (SELECT count(*) FROM gateway_tasks), + (SELECT count(*) FROM gateway_wallet_transactions WHERE gateway_user_id=$1::uuid), + (SELECT frozen_balance::text FROM gateway_wallet_accounts WHERE gateway_user_id=$1::uuid AND currency='resource')`, + smokeGatewayUserID, + ).Scan(&estimateSideEffectsAfter.Tasks, &estimateSideEffectsAfter.Transactions, &estimateSideEffectsAfter.Frozen); err != nil { + t.Fatalf("read estimate side effects after request: %v", err) + } + if estimateSideEffectsAfter != estimateSideEffectsBefore { + t.Fatalf("pricing estimate must be read-only, before=%+v after=%+v", estimateSideEffectsBefore, estimateSideEffectsAfter) + } + + idempotencyPayload := map[string]any{ + "model": pricingModel, "runMode": "simulation", "simulation": true, "simulationDurationMs": 5, + "messages": []map[string]any{{"role": "user", "content": "idempotent replay"}}, + } + idempotencyHeaders := map[string]string{"Idempotency-Key": "chat-replay-" + suffixText} + var idempotentFirst map[string]any + firstReplayHeaders := doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusOK, &idempotentFirst) + var idempotentSecond map[string]any + secondReplayHeaders := doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusOK, &idempotentSecond) + idempotentTaskID := firstReplayHeaders.Get("X-Gateway-Task-Id") + if idempotentTaskID == "" || secondReplayHeaders.Get("X-Gateway-Task-Id") != idempotentTaskID || secondReplayHeaders.Get("Idempotent-Replayed") != "true" { + t.Fatalf("non-stream replay headers first=%v second=%v", firstReplayHeaders, secondReplayHeaders) + } + var idempotentAttempts int + if err := testPool.QueryRow(ctx, `SELECT count(*) FROM gateway_task_attempts WHERE task_id=$1::uuid`, idempotentTaskID).Scan(&idempotentAttempts); err != nil { + t.Fatalf("count idempotent task attempts: %v", err) + } + if idempotentAttempts != 1 { + t.Fatalf("idempotent request called upstream %d times, want 1", idempotentAttempts) + } + idempotencyPayload["messages"] = []map[string]any{{"role": "user", "content": "different request"}} + doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, idempotencyPayload, idempotencyHeaders, http.StatusConflict, nil) + + streamPayload := map[string]any{ + "model": pricingModel, "runMode": "simulation", "simulation": true, "simulationDurationMs": 5, "stream": true, + "messages": []map[string]any{{"role": "user", "content": "stream replay"}}, + } + streamRaw, err := json.Marshal(streamPayload) + if err != nil { + t.Fatal(err) + } + streamKey := "stream-replay-" + suffixText + streamRequest, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/chat/completions", bytes.NewReader(streamRaw)) + if err != nil { + t.Fatal(err) + } + streamRequest.Header.Set("Authorization", "Bearer "+apiKeyResponse.Secret) + streamRequest.Header.Set("Content-Type", "application/json") + streamRequest.Header.Set("Idempotency-Key", streamKey) + streamResponse, err := http.DefaultClient.Do(streamRequest) + if err != nil { + t.Fatalf("execute first idempotent stream: %v", err) + } + _, _ = io.ReadAll(streamResponse.Body) + _ = streamResponse.Body.Close() + if streamResponse.StatusCode != http.StatusOK || streamResponse.Header.Get("X-Gateway-Task-Id") == "" { + t.Fatalf("first idempotent stream status=%d headers=%v", streamResponse.StatusCode, streamResponse.Header) + } + doJSONWithHeaders(t, server.URL, http.MethodPost, "/api/v1/chat/completions", apiKeyResponse.Secret, streamPayload, + map[string]string{"Idempotency-Key": streamKey}, http.StatusConflict, nil) var pricingTask struct { Task struct { ID string `json:"id"` @@ -1588,6 +1685,7 @@ WHERE m.platform_id = $1::uuid DatabaseURL: databaseURL, IdentityMode: "hybrid", JWTSecret: "test-secret", + BillingEngineMode: "enforce", TaskProgressCallbackEnabled: true, TaskProgressCallbackURL: "http://callback.local/task-progress", CORSAllowedOrigin: "*", -- 2.54.0 From 257ee09e589f56bb66b2de8d6cd43ccee393e26e Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 09:57:02 +0800 Subject: [PATCH 09/13] =?UTF-8?q?build(deps):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=E9=93=BE=E9=AB=98=E5=8D=B1=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pnpm audit 新增报告 Nx 构建链中的 axios、brace-expansion 和 js-yaml 高危漏洞。通过最小 overrides 将其固定到已修复版本，并同步锁文件。\n\n影响：仅调整开发与构建依赖解析，不改变生产运行时计费逻辑。\n\n验证：pnpm install --frozen-lockfile、pnpm audit --audit-level high、pnpm lint、pnpm test、pnpm build 及仓库完整 CI 门禁均已通过。 --- pnpm-lock.yaml | 65 ++++++++++++++++++++++++++++++++------------- pnpm-workspace.yaml | 4 ++- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0918eae..524987d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,15 +6,17 @@ settings: overrides: '@babel/core@<=7.29.0': 7.29.6 - dompurify@<=3.4.10: 3.4.11 - esbuild@>=0.27.3 <0.28.1: 0.28.1 - form-data@>=4.0.0 <4.0.6: 4.0.6 - js-yaml@<3.15.0: 4.1.1 - mermaid@>=11.0.0-alpha.1 <=11.14.0: 11.15.0 - minimatch@>=9.0.0 <9.0.7: 9.0.7 '@nx/js>picomatch': 4.0.5 '@nx/vite>picomatch': 4.0.5 '@nx/workspace>picomatch': 4.0.5 + axios@<1.18.0: 1.18.0 + brace-expansion@>=2.0.0 <2.1.2: 2.1.2 + dompurify@<=3.4.10: 3.4.11 + esbuild@>=0.27.3 <0.28.1: 0.28.1 + form-data@>=4.0.0 <4.0.6: 4.0.6 + js-yaml@<4.3.0: 4.3.0 + mermaid@>=11.0.0-alpha.1 <=11.14.0: 11.15.0 + minimatch@>=9.0.0 <9.0.7: 9.0.7 tmp@<0.2.7: 0.2.7 importers: @@ -2443,6 +2445,10 @@ packages: resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==} engines: {node: '>= 10.0.0'} + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} @@ -2491,8 +2497,8 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - axios@1.16.0: - resolution: {integrity: sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==} + axios@1.18.0: + resolution: {integrity: sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw==} babel-plugin-const-enum@1.2.0: resolution: {integrity: sha512-o1m/6iyyFnp9MRsK1dHF3bneqyf3AlM2q3A/YbgQr2pCat6B6XJVDv2TXqzfY2RYUi4mak6WAksSBPlyYGx9dg==} @@ -2553,8 +2559,8 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} - brace-expansion@2.1.0: - resolution: {integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==} + brace-expansion@2.1.2: + resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==} brace-expansion@5.0.7: resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} @@ -3184,6 +3190,10 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -3281,8 +3291,8 @@ packages: js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} - js-yaml@4.1.1: - resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + js-yaml@4.3.0: + resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} hasBin: true jsesc@3.1.0: @@ -5820,6 +5830,7 @@ snapshots: - '@swc-node/register' - '@swc/core' - debug + - supports-color '@phenomnomnominal/tsquery@5.0.1(typescript@5.9.3)': dependencies: @@ -7152,7 +7163,7 @@ snapshots: '@yarnpkg/parsers@3.0.2': dependencies: - js-yaml: 4.1.1 + js-yaml: 4.3.0 tslib: 2.8.1 '@zkochan/js-yaml@0.0.7': @@ -7161,6 +7172,12 @@ snapshots: address@1.2.2: {} + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + ajv@8.20.0: dependencies: fast-deep-equal: 3.1.3 @@ -7258,13 +7275,15 @@ snapshots: asynckit@0.4.0: {} - axios@1.16.0: + axios@1.18.0: dependencies: follow-redirects: 1.16.0 form-data: 4.0.6 + https-proxy-agent: 5.0.1 proxy-from-env: 2.1.0 transitivePeerDependencies: - debug + - supports-color babel-plugin-const-enum@1.2.0(@babel/core@7.29.6): dependencies: @@ -7336,7 +7355,7 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 - brace-expansion@2.1.0: + brace-expansion@2.1.2: dependencies: balanced-match: 1.0.2 @@ -7863,7 +7882,7 @@ snapshots: front-matter@4.0.2: dependencies: - js-yaml: 4.1.1 + js-yaml: 4.3.0 fs-constants@1.0.0: {} @@ -8054,6 +8073,13 @@ snapshots: html-void-elements@3.0.0: {} + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -8129,7 +8155,7 @@ snapshots: js-tokens@9.0.1: {} - js-yaml@4.1.1: + js-yaml@4.3.0: dependencies: argparse: 2.0.1 @@ -8677,7 +8703,7 @@ snapshots: minimatch@5.1.9: dependencies: - brace-expansion: 2.1.0 + brace-expansion: 2.1.2 minimatch@9.0.7: dependencies: @@ -8712,7 +8738,7 @@ snapshots: '@yarnpkg/lockfile': 1.1.0 '@yarnpkg/parsers': 3.0.2 '@zkochan/js-yaml': 0.0.7 - axios: 1.16.0 + axios: 1.18.0 chalk: 4.1.2 cli-cursor: 3.1.0 cli-spinners: 2.6.1 @@ -8756,6 +8782,7 @@ snapshots: '@nx/nx-win32-x64-msvc': 21.6.11 transitivePeerDependencies: - debug + - supports-color once@1.4.0: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fea2db6..012fa01 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,10 +8,12 @@ allowBuilds: overrides: '@babel/core@<=7.29.0': 7.29.6 + 'axios@<1.18.0': 1.18.0 + 'brace-expansion@>=2.0.0 <2.1.2': 2.1.2 'dompurify@<=3.4.10': 3.4.11 'esbuild@>=0.27.3 <0.28.1': 0.28.1 'form-data@>=4.0.0 <4.0.6': 4.0.6 - 'js-yaml@<3.15.0': 4.1.1 + 'js-yaml@<4.3.0': 4.3.0 'mermaid@>=11.0.0-alpha.1 <=11.14.0': 11.15.0 'minimatch@>=9.0.0 <9.0.7': 9.0.7 '@nx/js>picomatch': 4.0.5 -- 2.54.0 From 8beb8501fa2b2a89229fd11594ca51da432f791e Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 10:23:58 +0800 Subject: [PATCH 10/13] =?UTF-8?q?fix(billing):=20=E5=B0=81=E4=BD=8F?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=89=8D=E8=AE=A1=E8=B4=B9=E7=AB=9E=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 阻止上游提交状态不明的任务被租约接管后重复执行，并为人工复核保留可操作的结算记录。 将生产提交绑定到当前估价签名，统一复用预处理快照，并补强规则形状、定点溢出与历史规则兼容校验。 已通过 PostgreSQL 16 集成测试、Go 全量测试与静态检查、前端测试与构建、OpenAPI、依赖审计、镜像、迁移、流水线和 SemVer 门禁。 --- apps/api/internal/httpapi/handlers.go | 4 + apps/api/internal/httpapi/pricing_handlers.go | 7 + apps/api/internal/runner/execution_lease.go | 6 + apps/api/internal/runner/pricing.go | 7 +- apps/api/internal/runner/pricing_v2.go | 238 ++++++++++++++---- apps/api/internal/runner/pricing_v2_test.go | 43 +++- apps/api/internal/runner/queue_worker.go | 4 + apps/api/internal/runner/service.go | 60 ++++- .../store/billing_v2_integration_test.go | 143 +++++++++++ apps/api/internal/store/postgres.go | 2 + apps/api/internal/store/pricing_rules.go | 52 +++- apps/api/internal/store/pricing_rules_test.go | 27 ++ apps/api/internal/store/tasks_runtime.go | 120 ++++++++- .../0069_billing_correctness_v2.sql | 38 +++ .../src/pages/PlaygroundPage.pricing.test.ts | 8 + apps/web/src/pages/PlaygroundPage.tsx | 37 ++- .../pages/admin/BillingSettlementsPanel.tsx | 37 ++- docs/billing-flow-v2.md | 4 +- 18 files changed, 762 insertions(+), 75 deletions(-) create mode 100644 apps/api/internal/store/pricing_rules_test.go diff --git a/apps/api/internal/httpapi/handlers.go b/apps/api/internal/httpapi/handlers.go index 03dbae9..4d510f3 100644 --- a/apps/api/internal/httpapi/handlers.go +++ b/apps/api/internal/httpapi/handlers.go @@ -931,6 +931,10 @@ func (s *Server) estimatePricing(w http.ResponseWriter, r *http.Request) { writeErrorWithDetails(w, statusFromRunError(err), runErrorMessage(err), runErrorDetails(err), store.ModelCandidateErrorCode(err)) return } + if code := clients.ErrorCode(err); code == "bad_request" || code == "invalid_parameter" { + writeErrorWithDetails(w, http.StatusBadRequest, runErrorMessage(err), runErrorDetails(err), code) + return + } s.logger.Error("estimate pricing failed", "error", err) writeError(w, http.StatusInternalServerError, "estimate pricing failed") return diff --git a/apps/api/internal/httpapi/pricing_handlers.go b/apps/api/internal/httpapi/pricing_handlers.go index a0b8572..bc99e1c 100644 --- a/apps/api/internal/httpapi/pricing_handlers.go +++ b/apps/api/internal/httpapi/pricing_handlers.go @@ -165,6 +165,13 @@ func validPricingRuleSetInput(input store.PricingRuleSetInput) bool { default: return false } + calculator := strings.TrimSpace(rule.CalculatorType) + if calculator == "" { + calculator = store.DefaultEffectivePricingCalculator(rule.ResourceType) + } + if store.ValidateEffectivePricingRuleShape(strings.TrimSpace(rule.ResourceType), store.NormalizeEffectivePricingRuleUnit(rule.ResourceType, rule.Unit), calculator) != nil { + return false + } effectiveFrom, fromOK := pricingEffectiveTime(rule.EffectiveFrom) effectiveTo, toOK := pricingEffectiveTime(rule.EffectiveTo) if !fromOK || !toOK || (!effectiveFrom.IsZero() && !effectiveTo.IsZero() && !effectiveFrom.Before(effectiveTo)) { diff --git a/apps/api/internal/runner/execution_lease.go b/apps/api/internal/runner/execution_lease.go index 58e21de..6fa8aab 100644 --- a/apps/api/internal/runner/execution_lease.go +++ b/apps/api/internal/runner/execution_lease.go @@ -2,7 +2,10 @@ package runner import ( "context" + "errors" "time" + + "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" ) const ( @@ -19,6 +22,9 @@ func (s *Service) renewTaskExecutionLease(ctx context.Context, cancel context.Ca return case <-ticker.C: if err := s.store.RenewTaskExecutionLease(ctx, taskID, executionToken, taskExecutionLeaseTTL); err != nil { + if errors.Is(err, store.ErrTaskExecutionFinished) { + return + } s.logger.Warn("task execution lease lost", "taskID", taskID, "error_category", "task_execution_lease_lost") cancel() return diff --git a/apps/api/internal/runner/pricing.go b/apps/api/internal/runner/pricing.go index fb166d5..6366df2 100644 --- a/apps/api/internal/runner/pricing.go +++ b/apps/api/internal/runner/pricing.go @@ -38,8 +38,11 @@ func (s *Service) Estimate(ctx context.Context, kind string, model string, body } estimates := make([]candidateEstimate, 0, len(candidates)) for _, candidate := range candidates { - candidateBody := preprocessRequest(kind, cloneMap(body), candidate) - estimate, candidateErr := s.estimateCandidateV2(ctx, user, kind, candidateBody, candidate) + preprocessing := s.preprocessRequestWithScripts(ctx, kind, body, candidate) + if preprocessing.Err != nil { + return EstimateResult{}, parameterPreprocessClientError(preprocessing.Err) + } + estimate, candidateErr := s.estimateCandidateV2(ctx, user, kind, preprocessing.Body, candidate) if candidateErr != nil { return EstimateResult{}, candidateErr } diff --git a/apps/api/internal/runner/pricing_v2.go b/apps/api/internal/runner/pricing_v2.go index 95c42db..53c32db 100644 --- a/apps/api/internal/runner/pricing_v2.go +++ b/apps/api/internal/runner/pricing_v2.go @@ -20,9 +20,11 @@ import ( const ( pricingVersionV2 = "effective-pricing-v2" fixedScale = int64(1_000_000_000) + maxFixedAmount = fixedAmount(math.MaxInt64) ) var ErrPricingUnavailable = errors.New("pricing unavailable") +var errFixedAmountOverflow = errors.New("fixed amount overflow") type PricingUnavailableError struct { Reason string @@ -129,11 +131,11 @@ func fixedAmountFromAny(value any) (fixedAmount, error) { case json.Number: return parseFixedAmount(typed.String()) case int: - return fixedAmount(int64(typed) * fixedScale), nil + return parseFixedAmount(strconv.FormatInt(int64(typed), 10)) case int64: - return fixedAmount(typed * fixedScale), nil + return parseFixedAmount(strconv.FormatInt(typed, 10)) case int32: - return fixedAmount(int64(typed) * fixedScale), nil + return parseFixedAmount(strconv.FormatInt(int64(typed), 10)) case float64: if math.IsNaN(typed) || math.IsInf(typed, 0) { return 0, fmt.Errorf("non-finite decimal") @@ -159,26 +161,37 @@ func (a fixedAmount) String() string { func (a fixedAmount) Float64() float64 { return float64(a) / float64(fixedScale) } func (a fixedAmount) IsZero() bool { return a == 0 } -func (a fixedAmount) Add(other fixedAmount) fixedAmount { return a + other } - -func (a fixedAmount) MulInt(multiplier int) fixedAmount { - return fixedAmount(int64(a) * int64(multiplier)) +func addFixedAmounts(left fixedAmount, right fixedAmount) (fixedAmount, error) { + value := new(big.Int).Add(big.NewInt(int64(left)), big.NewInt(int64(right))) + return fixedAmountFromBigInt(value) } -func (a fixedAmount) Mul(other fixedAmount) fixedAmount { - product := new(big.Int).Mul(big.NewInt(int64(a)), big.NewInt(int64(other))) - return fixedAmount(roundBigIntRatio(product, big.NewInt(fixedScale))) +func multiplyFixedAmountByInt(amount fixedAmount, multiplier int) (fixedAmount, error) { + value := new(big.Int).Mul(big.NewInt(int64(amount)), big.NewInt(int64(multiplier))) + return fixedAmountFromBigInt(value) } -func (a fixedAmount) MulRatio(numerator int, denominator int) fixedAmount { +func multiplyFixedAmounts(left fixedAmount, right fixedAmount) (fixedAmount, error) { + product := new(big.Int).Mul(big.NewInt(int64(left)), big.NewInt(int64(right))) + return fixedAmountFromBigInt(roundBigIntRatio(product, big.NewInt(fixedScale))) +} + +func multiplyFixedAmountRatio(amount fixedAmount, numerator int, denominator int) (fixedAmount, error) { if denominator == 0 { - return 0 + return 0, fmt.Errorf("division by zero") } - product := new(big.Int).Mul(big.NewInt(int64(a)), big.NewInt(int64(numerator))) - return fixedAmount(roundBigIntRatio(product, big.NewInt(int64(denominator)))) + product := new(big.Int).Mul(big.NewInt(int64(amount)), big.NewInt(int64(numerator))) + return fixedAmountFromBigInt(roundBigIntRatio(product, big.NewInt(int64(denominator)))) } -func roundBigIntRatio(numerator *big.Int, denominator *big.Int) int64 { +func fixedAmountFromBigInt(value *big.Int) (fixedAmount, error) { + if value == nil || !value.IsInt64() { + return 0, errFixedAmountOverflow + } + return fixedAmount(value.Int64()), nil +} + +func roundBigIntRatio(numerator *big.Int, denominator *big.Int) *big.Int { negative := numerator.Sign() < 0 absolute := new(big.Int).Abs(new(big.Int).Set(numerator)) quotient, remainder := new(big.Int), new(big.Int) @@ -189,7 +202,7 @@ func roundBigIntRatio(numerator *big.Int, denominator *big.Int) int64 { if negative { quotient.Neg(quotient) } - return quotient.Int64() + return quotient } type resolvedPricing struct { @@ -248,6 +261,105 @@ func (pricing resolvedPricing) price(resource string, keys ...string) (fixedAmou return 0, false, nil } +func (pricing resolvedPricing) weight(resource string, key string, name string) (fixedAmount, error) { + if strings.TrimSpace(name) == "" { + return fixedAmount(fixedScale), nil + } + keys := weightKeyAliases(key) + names := weightValueAliases(key, name) + value, found := pricingWeightValue(pricing.Config, resource, keys, names) + if !found { + return fixedAmount(fixedScale), nil + } + weight, err := fixedAmountFromAny(value) + if err != nil || weight <= 0 { + reason := fmt.Sprintf("invalid %s weight for %s", key, name) + if err != nil { + reason += ": " + err.Error() + } + return 0, &PricingUnavailableError{Reason: reason, ResourceType: resource, RuleSetID: pricing.RuleSetID} + } + return weight, nil +} + +func (pricing resolvedPricing) calculate(resource string, base fixedAmount, integerFactors []int, fixedFactors ...fixedAmount) (fixedAmount, error) { + amount := base + var err error + for _, factor := range integerFactors { + amount, err = multiplyFixedAmountByInt(amount, factor) + if err != nil { + return 0, pricing.calculationError(resource, err) + } + } + for _, factor := range fixedFactors { + amount, err = multiplyFixedAmounts(amount, factor) + if err != nil { + return 0, pricing.calculationError(resource, err) + } + } + return amount, nil +} + +func (pricing resolvedPricing) calculateRatio(resource string, base fixedAmount, numerator int, denominator int, fixedFactors ...fixedAmount) (fixedAmount, error) { + amount, err := multiplyFixedAmountRatio(base, numerator, denominator) + if err != nil { + return 0, pricing.calculationError(resource, err) + } + return pricing.calculate(resource, amount, nil, fixedFactors...) +} + +func (pricing resolvedPricing) add(resource string, left fixedAmount, right fixedAmount) (fixedAmount, error) { + amount, err := addFixedAmounts(left, right) + if err != nil { + return 0, pricing.calculationError(resource, err) + } + return amount, nil +} + +func (pricing resolvedPricing) calculationError(resource string, err error) error { + return &PricingUnavailableError{Reason: "pricing calculation failed: " + err.Error(), ResourceType: resource, RuleSetID: pricing.RuleSetID} +} + +func pricingWeightValue(config map[string]any, resource string, keys []string, names []string) (any, bool) { + if value, ok := pricingWeightValueFromConfig(config, keys, names); ok { + return value, true + } + resourceConfig, _ := config[resource].(map[string]any) + if len(resourceConfig) == 0 && resource == "image_edit" { + resourceConfig, _ = config["image"].(map[string]any) + } + return pricingWeightValueFromConfig(resourceConfig, keys, names) +} + +func pricingWeightValueFromConfig(config map[string]any, keys []string, names []string) (any, bool) { + if len(config) == 0 { + return nil, false + } + for _, key := range keys { + weights, _ := config[key].(map[string]any) + for _, name := range names { + if value, ok := weights[name]; ok { + return value, true + } + } + } + dynamic, _ := config["dynamicWeight"].(map[string]any) + for _, name := range names { + if value, ok := dynamic[name]; ok { + return value, true + } + } + for _, key := range keys { + weights, _ := dynamic[key].(map[string]any) + for _, name := range names { + if value, ok := weights[name]; ok { + return value, true + } + } + } + return nil, false +} + type candidateEstimate struct { Items []any Amount fixedAmount @@ -474,8 +586,14 @@ func (s *Service) billingsWithResolvedPricingV2( items := make([]any, 0, 3) total := fixedAmount(0) if uncachedInputTokens > 0 || cachedInputTokens == 0 { - amount := inputPrice.MulRatio(uncachedInputTokens, 1000).Mul(discount) - total = total.Add(amount) + amount, calculationErr := pricing.calculateRatio("text_input", inputPrice, uncachedInputTokens, 1000, discount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } + total, calculationErr = pricing.add("text", total, amount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } items = append(items, buildLine("text_input", "1k_tokens", uncachedInputTokens, amount, map[string]any{ "inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens, "cachedInputTokens": cachedInputTokens, "pricePer1k": inputPrice.Float64(), @@ -486,8 +604,14 @@ func (s *Service) billingsWithResolvedPricingV2( if priceErr != nil { return nil, 0, resolvedPricing{}, priceErr } - amount := cachedPrice.MulRatio(cachedInputTokens, 1000).Mul(discount) - total = total.Add(amount) + amount, calculationErr := pricing.calculateRatio("text_cached_input", cachedPrice, cachedInputTokens, 1000, discount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } + total, calculationErr = pricing.add("text", total, amount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } items = append(items, buildLine("text_cached_input", "1k_tokens", cachedInputTokens, amount, map[string]any{ "inputTokens": inputTokens, "uncachedInputTokens": uncachedInputTokens, "cachedInputTokens": cachedInputTokens, "pricePer1k": cachedPrice.Float64(), @@ -498,8 +622,14 @@ func (s *Service) billingsWithResolvedPricingV2( if priceErr != nil { return nil, 0, resolvedPricing{}, priceErr } - amount := outputPrice.MulRatio(outputTokens, 1000).Mul(discount) - total = total.Add(amount) + amount, calculationErr := pricing.calculateRatio("text_output", outputPrice, outputTokens, 1000, discount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } + total, calculationErr = pricing.add("text", total, amount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } items = append(items, buildLine("text_output", "1k_tokens", outputTokens, amount, map[string]any{"pricePer1k": outputPrice.Float64()})) } return items, total, pricing, nil @@ -524,12 +654,26 @@ func (s *Service) billingsWithResolvedPricingV2( if priceErr != nil { return nil, 0, resolvedPricing{}, priceErr } - amount := price.MulInt(count).MulInt(durationUnits). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))))). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "audioWeights", boolWeightKey(audioEnabled)))). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "referenceVideoWeights", boolWeightKey(requestHasReferenceVideo(body))))). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "voiceSpecifiedWeights", boolWeightKey(requestHasVoiceID(body, audioEnabled))))). - Mul(discount) + resolutionWeight, weightErr := pricing.weight(resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + audioWeight, weightErr := pricing.weight(resource, "audioWeights", boolWeightKey(audioEnabled)) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + referenceVideoWeight, weightErr := pricing.weight(resource, "referenceVideoWeights", boolWeightKey(requestHasReferenceVideo(body))) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + voiceWeight, weightErr := pricing.weight(resource, "voiceSpecifiedWeights", boolWeightKey(requestHasVoiceID(body, audioEnabled))) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + amount, calculationErr := pricing.calculate(resource, price, []int{count, durationUnits}, resolutionWeight, audioWeight, referenceVideoWeight, voiceWeight, discount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } item := buildLine(resource, unit, count*durationUnits, amount, map[string]any{ "count": count, "audio": audioEnabled, "audioSource": audioSource, "durationSeconds": duration, "durationSource": durationSource, @@ -555,14 +699,32 @@ func (s *Service) billingsWithResolvedPricingV2( if err != nil { return nil, 0, resolvedPricing{}, err } - amount := price.MulInt(count) + amount, calculationErr := pricing.calculate(resource, price, []int{count}) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } if resource == "image" || resource == "image_edit" { - amount = amount. - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "qualityWeights", stringFromMap(body, "quality")))). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "sizeWeights", stringFromMap(body, "size")))). - Mul(fixedWeight(resourceWeight(pricing.Config, resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))))) + qualityWeight, weightErr := pricing.weight(resource, "qualityWeights", stringFromMap(body, "quality")) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + sizeWeight, weightErr := pricing.weight(resource, "sizeWeights", stringFromMap(body, "size")) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + resolutionWeight, weightErr := pricing.weight(resource, "resolutionWeights", firstNonEmptyString(stringFromMap(body, "resolution"), stringFromMap(body, "size"))) + if weightErr != nil { + return nil, 0, resolvedPricing{}, weightErr + } + amount, calculationErr = pricing.calculate(resource, amount, nil, qualityWeight, sizeWeight, resolutionWeight) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr + } + } + amount, calculationErr = pricing.calculate(resource, amount, nil, discount) + if calculationErr != nil { + return nil, 0, resolvedPricing{}, calculationErr } - amount = amount.Mul(discount) return []any{buildLine(resource, unit, count, amount, nil)}, amount, pricing, nil } @@ -580,14 +742,6 @@ func (pricing resolvedPricing) requiredTextPrice(resource string, keys ...string return 0, &PricingUnavailableError{Reason: "missing, invalid, or not explicitly free", ResourceType: resource, RuleSetID: pricing.RuleSetID} } -func fixedWeight(value float64) fixedAmount { - weight, err := fixedAmountFromAny(value) - if err != nil || weight <= 0 { - return fixedAmount(fixedScale) - } - return weight -} - func maximumCandidateEstimate(estimates []candidateEstimate) (candidateEstimate, error) { if len(estimates) == 0 { return candidateEstimate{}, &PricingUnavailableError{Reason: "no candidate has effective pricing"} diff --git a/apps/api/internal/runner/pricing_v2_test.go b/apps/api/internal/runner/pricing_v2_test.go index ea16827..dd2acad 100644 --- a/apps/api/internal/runner/pricing_v2_test.go +++ b/apps/api/internal/runner/pricing_v2_test.go @@ -1,6 +1,8 @@ package runner import ( + "errors" + "math" "testing" "github.com/easyai/easyai-ai-gateway/apps/api/internal/store" @@ -16,11 +18,29 @@ func TestFixedAmountPreservesNineDecimalPlaces(t *testing.T) { } price := mustFixedAmount(t, "0.000000001") - if got, want := price.MulInt(3).String(), "0.000000003"; got != want { + product, err := multiplyFixedAmountByInt(price, 3) + if err != nil { + t.Fatalf("multiply fixed amount: %v", err) + } + if got, want := product.String(), "0.000000003"; got != want { t.Fatalf("nano amount multiplication=%q, want %q", got, want) } } +func TestFixedAmountOperationsRejectOverflow(t *testing.T) { + maximum := fixedAmount(math.MaxInt64) + if _, err := multiplyFixedAmountByInt(maximum, 2); !errors.Is(err, errFixedAmountOverflow) { + t.Fatalf("multiply overflow error=%v", err) + } + if _, err := addFixedAmounts(maximum, 1); !errors.Is(err, errFixedAmountOverflow) { + t.Fatalf("add overflow error=%v", err) + } + pricing := resolvedPricing{RuleSetID: "overflow-rule"} + if _, err := pricing.calculate("image", maximum, []int{2}); !isPricingUnavailable(err) { + t.Fatalf("pricing overflow should be unavailable: %v", err) + } +} + func TestEstimatedOutputTokensUsesAliasesAndCapabilityFallback(t *testing.T) { candidate := store.RuntimeModelCandidate{ ModelType: "text_generate", @@ -97,6 +117,27 @@ func TestPricingAvailabilityRequiresExplicitFree(t *testing.T) { } } +func TestPricingWeightsUseFixedPrecisionAndRejectInvalidValues(t *testing.T) { + pricing := resolvedPricing{Config: map[string]any{ + "image": map[string]any{ + "dynamicWeight": map[string]any{ + "qualityFactors": map[string]any{"high": "1.123456789", "broken": 0}, + }, + }, + }} + weight, err := pricing.weight("image", "qualityWeights", "high") + if err != nil || weight.String() != "1.123456789" { + t.Fatalf("exact weight=%s err=%v", weight.String(), err) + } + if _, err := pricing.weight("image", "qualityWeights", "broken"); !isPricingUnavailable(err) { + t.Fatalf("invalid configured weight should make pricing unavailable: %v", err) + } + defaultWeight, err := pricing.weight("image", "qualityWeights", "unconfigured") + if err != nil || defaultWeight.String() != "1.000000000" { + t.Fatalf("default weight=%s err=%v", defaultWeight.String(), err) + } +} + func mustFixedAmount(t *testing.T, value string) fixedAmount { t.Helper() amount, err := parseFixedAmount(value) diff --git a/apps/api/internal/runner/queue_worker.go b/apps/api/internal/runner/queue_worker.go index 17880b8..311821e 100644 --- a/apps/api/internal/runner/queue_worker.go +++ b/apps/api/internal/runner/queue_worker.go @@ -49,6 +49,10 @@ func (w *asyncTaskWorker) Work(ctx context.Context, job *river.Job[asyncTaskArgs w.service.logger.Debug("river async task execution lease already held", "taskID", task.ID, "riverJobID", job.ID) return nil } + if errors.Is(runErr, store.ErrTaskExecutionManualReview) { + w.service.logger.Warn("river async task moved to manual review after ambiguous upstream submission", "taskID", task.ID, "riverJobID", job.ID) + return nil + } var queuedErr *TaskQueuedError if errors.As(runErr, &queuedErr) { return river.JobSnooze(queuedErr.Delay) diff --git a/apps/api/internal/runner/service.go b/apps/api/internal/runner/service.go index 2f2b29f..2f94a38 100644 --- a/apps/api/internal/runner/service.go +++ b/apps/api/internal/runner/service.go @@ -302,6 +302,7 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, } } pricingByCandidate := map[string]resolvedPricing{} + preprocessingByCandidate := map[string]parameterPreprocessResult{} reservationBillings := []any(nil) reservationPricingSnapshot := map[string]any(nil) if task.RunMode == "production" { @@ -316,9 +317,17 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, } estimates := make([]candidateEstimate, 0, len(candidates)) var pricingErr error + var preprocessingErr error + var preprocessingCandidate store.RuntimeModelCandidate for _, candidate := range candidates { - pricingBody := preprocessRequest(task.Kind, cloneMap(body), candidate) - estimate, estimateErr := s.estimateCandidateV2(ctx, user, task.Kind, pricingBody, candidate) + preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate) + preprocessingByCandidate[pricingCandidateKey(candidate)] = preprocessing + if preprocessing.Err != nil { + preprocessingErr = parameterPreprocessClientError(preprocessing.Err) + preprocessingCandidate = candidate + break + } + estimate, estimateErr := s.estimateCandidateV2(ctx, user, task.Kind, preprocessing.Body, candidate) if estimateErr != nil { pricingErr = estimateErr if billingMode == "enforce" { @@ -329,8 +338,23 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, estimates = append(estimates, estimate) pricingByCandidate[pricingCandidateKey(candidate)] = estimate.Pricing } + if preprocessingErr != nil { + preprocessing := preprocessingByCandidate[pricingCandidateKey(preprocessingCandidate)] + s.recordFailedAttempt(ctx, failedAttemptRecord{ + Task: task, Body: preprocessing.Body, Candidate: &preprocessingCandidate, + AttemptNo: task.AttemptCount + 1, Code: clients.ErrorCode(preprocessingErr), Cause: preprocessingErr, + Simulated: false, Scope: "parameter_preprocessing", Reason: "parameter_preprocessing_failed", + ExtraMetrics: []map[string]any{parameterPreprocessingMetrics(preprocessing.Log)}, Preprocessing: &preprocessing.Log, + ModelType: preprocessingCandidate.ModelType, + }) + failed, finishErr := s.failTask(ctx, task.ID, task.ExecutionToken, clients.ErrorCode(preprocessingErr), preprocessingErr.Error(), false, preprocessingErr, parameterPreprocessingMetrics(preprocessing.Log)) + if finishErr != nil { + return Result{}, finishErr + } + return Result{Task: failed, Output: failed.Result}, preprocessingErr + } if billingMode == "observe" { - legacyItems, legacyAmount := s.maximumLegacyCandidateEstimate(ctx, user, task.Kind, body, candidates) + legacyItems, legacyAmount := s.maximumLegacyCandidateEstimate(ctx, user, task.Kind, body, candidates, preprocessingByCandidate) reservationBillings = legacyItems candidateSnapshots := make([]any, 0, len(estimates)) for _, estimate := range estimates { @@ -396,7 +420,10 @@ func (s *Service) executeWithToken(ctx context.Context, task store.GatewayTask, } }() if len(candidates) > 0 { - preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidates[0]) + preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidates[0])] + if !ok { + preprocessing = s.preprocessRequestWithScripts(ctx, task.Kind, body, candidates[0]) + } firstCandidateBody = preprocessing.Body firstPreprocessing = preprocessing.Log normalizedModelType = candidates[0].ModelType @@ -472,7 +499,10 @@ candidatesLoop: var candidateErr error for clientAttempt := 1; clientAttempt <= clientAttempts; clientAttempt++ { nextAttemptNo := attemptNo + 1 - preprocessing := s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate) + preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)] + if !ok { + preprocessing = s.preprocessRequestWithScripts(ctx, task.Kind, body, candidate) + } preprocessingLog := preprocessing.Log lastPreprocessing = &preprocessingLog if preprocessing.Err != nil { @@ -768,11 +798,14 @@ func normalizedBillingEngineMode(value string) string { } } -func (s *Service) maximumLegacyCandidateEstimate(ctx context.Context, user *auth.User, kind string, body map[string]any, candidates []store.RuntimeModelCandidate) ([]any, fixedAmount) { +func (s *Service) maximumLegacyCandidateEstimate(ctx context.Context, user *auth.User, kind string, body map[string]any, candidates []store.RuntimeModelCandidate, preprocessingByCandidate map[string]parameterPreprocessResult) ([]any, fixedAmount) { var maximumItems []any maximumAmount := fixedAmount(0) for index, candidate := range candidates { candidateBody := preprocessRequest(kind, cloneMap(body), candidate) + if preprocessing, ok := preprocessingByCandidate[pricingCandidateKey(candidate)]; ok && preprocessing.Err == nil { + candidateBody = preprocessing.Body + } items := s.estimatedBillings(ctx, user, kind, candidateBody, candidate) amount := billingItemsFixedTotal(items) if index == 0 || amount > maximumAmount { @@ -792,7 +825,11 @@ func billingItemsFixedTotal(items []any) fixedAmount { } amount, err := fixedAmountFromAny(line["amount"]) if err == nil && amount > 0 { - total = total.Add(amount) + next, addErr := addFixedAmounts(total, amount) + if addErr != nil { + return maxFixedAmount + } + total = next } } return total @@ -925,6 +962,11 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user PreviousResponseTurns: responseExecution.PreviousTurns, }) callFinishedAt := time.Now() + if err == nil { + if markErr := s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received"); markErr != nil { + return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr} + } + } if response.ResponseStartedAt.IsZero() { response.ResponseStartedAt = callStartedAt } @@ -939,7 +981,9 @@ func (s *Service) runCandidate(ctx context.Context, task store.GatewayTask, user } if err != nil { if clients.ErrorResponseMetadata(err).StatusCode > 0 { - _ = s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received") + if markErr := s.store.SetAttemptUpstreamSubmissionStatus(context.WithoutCancel(ctx), attemptID, "response_received"); markErr != nil { + return clients.Response{}, &upstreamSubmissionUnknownError{AttemptID: attemptID, Cause: markErr} + } } retryable := clients.IsRetryable(err) requestID, metrics, responseStartedAt, responseFinishedAt, responseDurationMS := failureMetrics(err, simulated) diff --git a/apps/api/internal/store/billing_v2_integration_test.go b/apps/api/internal/store/billing_v2_integration_test.go index e078029..30ed972 100644 --- a/apps/api/internal/store/billing_v2_integration_test.go +++ b/apps/api/internal/store/billing_v2_integration_test.go @@ -62,6 +62,149 @@ func TestTaskIdempotencyAndExecutionLease(t *testing.T) { if err != nil || finished.ErrorCode != "new_worker" { t.Fatalf("new worker terminal task=%+v err=%v", finished, err) } + if err := db.RenewTaskExecutionLease(ctx, created.Task.ID, secondToken, 5*time.Minute); !errors.Is(err, ErrTaskExecutionFinished) { + t.Fatalf("terminal task renewal error=%v", err) + } +} + +func TestExpiredExecutionLeaseDoesNotReplayAmbiguousUpstreamSubmission(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + tenantID, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-review-" + uuid.NewString(), GatewayUserID: gatewayUserID, GatewayTenantID: tenantID} + created, err := db.CreateTask(ctx, CreateTaskInput{ + Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", + Request: map[string]any{"model": "billing-v2-model"}, + }, user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID) + }) + + firstToken := uuid.NewString() + if _, err := db.ClaimTaskExecution(ctx, created.ID, firstToken, 5*time.Minute); err != nil { + t.Fatal(err) + } + attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{ + TaskID: created.ID, AttemptNo: 1, Status: "running", RequestSnapshot: map[string]any{"model": created.Model}, + }) + if err != nil { + t.Fatal(err) + } + if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil { + t.Fatal(err) + } + if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil { + t.Fatal(err) + } + if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); !errors.Is(err, ErrTaskExecutionManualReview) { + t.Fatalf("ambiguous submission takeover error=%v", err) + } + + review, err := db.GetTask(ctx, created.ID) + if err != nil { + t.Fatal(err) + } + if review.Status != "failed" || review.BillingStatus != "manual_review" || review.ErrorCode != "upstream_submission_unknown" { + t.Fatalf("manual review task=%+v", review) + } + var outboxStatus string + var outboxAction string + var reviewReason string + if err := db.pool.QueryRow(ctx, ` +SELECT status, action, COALESCE(manual_review_reason, '') +FROM settlement_outbox +WHERE task_id=$1::uuid AND event_type='task.billing.review'`, created.ID).Scan(&outboxStatus, &outboxAction, &reviewReason); err != nil { + t.Fatal(err) + } + if outboxStatus != "manual_review" || outboxAction != "release" || reviewReason != "upstream_submission_unknown" { + t.Fatalf("review outbox status=%s action=%s reason=%s", outboxStatus, outboxAction, reviewReason) + } +} + +func TestExpiredExecutionLeaseCanResumeAfterKnownRejectedResponse(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + _, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-known-response-" + uuid.NewString(), GatewayUserID: gatewayUserID} + created, err := db.CreateTask(ctx, CreateTaskInput{ + Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", + Request: map[string]any{"model": "billing-v2-model"}, + }, user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID) + }) + + if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil { + t.Fatal(err) + } + attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"}) + if err != nil { + t.Fatal(err) + } + if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "response_received"); err != nil { + t.Fatal(err) + } + if err := db.FinishTaskAttempt(ctx, FinishTaskAttemptInput{AttemptID: attemptID, Status: "failed", ErrorCode: "upstream_rejected"}); err != nil { + t.Fatal(err) + } + if _, err := db.pool.Exec(ctx, `UPDATE gateway_tasks SET execution_lease_expires_at=now()-interval '1 second' WHERE id=$1::uuid`, created.ID); err != nil { + t.Fatal(err) + } + if _, err := db.ClaimTaskExecution(ctx, created.ID, uuid.NewString(), 5*time.Minute); err != nil { + t.Fatalf("known rejected response should remain retryable: %v", err) + } +} + +func TestFinishTaskManualReviewCreatesVisibleBillingRecord(t *testing.T) { + db := billingV2IntegrationStore(t) + ctx := context.Background() + _, gatewayUserID := seedWalletReservationUser(t, ctx, db) + user := &auth.User{ID: "billing-direct-review-" + uuid.NewString(), GatewayUserID: gatewayUserID} + created, err := db.CreateTask(ctx, CreateTaskInput{ + Kind: "images.generations", Model: "billing-v2-model", RunMode: "production", + Request: map[string]any{"model": "billing-v2-model"}, + }, user) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = db.pool.Exec(context.Background(), `DELETE FROM gateway_tasks WHERE id=$1::uuid`, created.ID) + }) + token := uuid.NewString() + if _, err := db.ClaimTaskExecution(ctx, created.ID, token, 5*time.Minute); err != nil { + t.Fatal(err) + } + attemptID, err := db.CreateTaskAttempt(ctx, CreateTaskAttemptInput{TaskID: created.ID, AttemptNo: 1, Status: "running"}) + if err != nil { + t.Fatal(err) + } + if err := db.SetAttemptUpstreamSubmissionStatus(ctx, attemptID, "submitting"); err != nil { + t.Fatal(err) + } + if _, err := db.FinishTaskManualReview(ctx, FinishTaskManualReviewInput{ + TaskID: created.ID, ExecutionToken: token, AttemptID: attemptID, TaskStatus: "failed", + Code: "upstream_submission_unknown", Message: "upstream submission result is unknown", + }); err != nil { + t.Fatal(err) + } + var visible bool + if err := db.pool.QueryRow(ctx, ` +SELECT EXISTS ( + SELECT 1 FROM settlement_outbox + WHERE task_id=$1::uuid AND status='manual_review' + AND action='release' AND manual_review_reason='upstream_submission_unknown' +)`, created.ID).Scan(&visible); err != nil { + t.Fatal(err) + } + if !visible { + t.Fatal("manual review billing record is not visible in settlement outbox") + } } func TestBillingSettlementStaleTakeoverDebitsExactlyOnce(t *testing.T) { diff --git a/apps/api/internal/store/postgres.go b/apps/api/internal/store/postgres.go index b3d7f93..8de239e 100644 --- a/apps/api/internal/store/postgres.go +++ b/apps/api/internal/store/postgres.go @@ -59,6 +59,8 @@ var ( ErrIdempotencyKeyReused = errors.New("idempotency key was reused for a different request") ErrTaskExecutionLeaseUnavailable = errors.New("task execution lease is unavailable") ErrTaskExecutionLeaseLost = errors.New("task execution lease was lost") + ErrTaskExecutionFinished = errors.New("task execution already finished") + ErrTaskExecutionManualReview = errors.New("task execution requires manual review") ErrProtectedDefault = errors.New("protected default resource cannot be deleted") ErrUserAlreadyExists = errors.New("user already exists") ErrWeakPassword = errors.New("password must be at least 8 characters") diff --git a/apps/api/internal/store/pricing_rules.go b/apps/api/internal/store/pricing_rules.go index a3136cc..bbff1ad 100644 --- a/apps/api/internal/store/pricing_rules.go +++ b/apps/api/internal/store/pricing_rules.go @@ -335,6 +335,9 @@ ORDER BY priority ASC, resource_type ASC, rule_key ASC`, id) default: return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s uses unsupported calculator %s", ruleKey, calculatorType) } + if err := ValidateEffectivePricingRuleShape(resourceType, unit, calculatorType); err != nil { + return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s: %w", ruleKey, err) + } if strings.HasPrefix(strings.TrimSpace(basePrice), "-") { return EffectivePricingConfig{}, fmt.Errorf("pricing rule %s has a negative base price", ruleKey) } @@ -372,6 +375,51 @@ ORDER BY priority ASC, resource_type ASC, rule_key ASC`, id) }, nil } +func ValidateEffectivePricingRuleShape(resourceType string, unit string, calculatorType string) error { + unit = NormalizeEffectivePricingRuleUnit(resourceType, unit) + if strings.TrimSpace(calculatorType) == "" { + calculatorType = DefaultEffectivePricingCalculator(resourceType) + } + allowed := false + switch resourceType { + case "text_input", "text_cached_input", "text_output", "text_total": + allowed = unit == "1k_tokens" && calculatorType == "token_usage" + case "image", "image_edit": + allowed = unit == "image" && calculatorType == "unit_weight" + case "video": + allowed = unit == "5s" && calculatorType == "duration_weight" + case "music": + allowed = (unit == "song" || unit == "item") && calculatorType == "unit_weight" + case "audio": + allowed = unit == "character" && calculatorType == "unit_weight" + default: + return nil + } + if !allowed { + return fmt.Errorf("unit %q and calculator %q do not match resource %q", unit, calculatorType, resourceType) + } + return nil +} + +func NormalizeEffectivePricingRuleUnit(resourceType string, unit string) string { + unit = strings.TrimSpace(unit) + if resourceType == "video" && unit == "video" { + return "5s" + } + return unit +} + +func DefaultEffectivePricingCalculator(resourceType string) string { + switch strings.TrimSpace(resourceType) { + case "text_input", "text_cached_input", "text_output", "text_total": + return "token_usage" + case "video": + return "duration_weight" + default: + return "unit_weight" + } +} + func addPricingRuleToConfig(config map[string]any, resourceType string, basePrice string, dynamicWeight map[string]any, formulaConfig map[string]any) { resourceConfig := map[string]any{"basePrice": basePrice} if len(dynamicWeight) > 0 { @@ -591,7 +639,7 @@ func normalizePricingRule(input PricingRuleInput, index int, defaultCurrency str input.RuleKey = strings.TrimSpace(input.RuleKey) input.DisplayName = strings.TrimSpace(input.DisplayName) input.ResourceType = strings.TrimSpace(input.ResourceType) - input.Unit = strings.TrimSpace(input.Unit) + input.Unit = NormalizeEffectivePricingRuleUnit(input.ResourceType, input.Unit) input.Currency = strings.TrimSpace(input.Currency) input.CalculatorType = strings.TrimSpace(input.CalculatorType) input.Status = strings.TrimSpace(input.Status) @@ -608,7 +656,7 @@ func normalizePricingRule(input PricingRuleInput, index int, defaultCurrency str input.Currency = defaultCurrency } if input.CalculatorType == "" { - input.CalculatorType = "unit_weight" + input.CalculatorType = DefaultEffectivePricingCalculator(input.ResourceType) } if input.Priority == 0 { input.Priority = (index + 1) * 10 diff --git a/apps/api/internal/store/pricing_rules_test.go b/apps/api/internal/store/pricing_rules_test.go new file mode 100644 index 0000000..4c538cd --- /dev/null +++ b/apps/api/internal/store/pricing_rules_test.go @@ -0,0 +1,27 @@ +package store + +import "testing" + +func TestValidateEffectivePricingRuleShape(t *testing.T) { + tests := []struct { + name string + resource string + unit string + calculator string + valid bool + }{ + {name: "text", resource: "text_input", unit: "1k_tokens", calculator: "token_usage", valid: true}, + {name: "image", resource: "image", unit: "image", calculator: "unit_weight", valid: true}, + {name: "video", resource: "video", unit: "5s", calculator: "duration_weight", valid: true}, + {name: "wrong video unit", resource: "video", unit: "second", calculator: "duration_weight"}, + {name: "wrong image calculator", resource: "image", unit: "image", calculator: "token_usage"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := ValidateEffectivePricingRuleShape(test.resource, test.unit, test.calculator) + if (err == nil) != test.valid { + t.Fatalf("valid=%v err=%v", test.valid, err) + } + }) + } +} diff --git a/apps/api/internal/store/tasks_runtime.go b/apps/api/internal/store/tasks_runtime.go index 0b55e76..fff569a 100644 --- a/apps/api/internal/store/tasks_runtime.go +++ b/apps/api/internal/store/tasks_runtime.go @@ -157,7 +157,83 @@ func (s *Store) ClaimTaskExecution(ctx context.Context, taskID string, execution if leaseTTL <= 0 { leaseTTL = 5 * time.Minute } - task, err := scanGatewayTask(s.pool.QueryRow(ctx, ` + var task GatewayTask + manualReview := false + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var queuedReady bool + var runningExpired bool + var production bool + var hasGatewayUser bool + if err := tx.QueryRow(ctx, ` +SELECT status = 'queued' AND next_run_at <= now(), + status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now()), + run_mode = 'production', + gateway_user_id IS NOT NULL +FROM gateway_tasks +WHERE id = $1::uuid +FOR UPDATE`, taskID).Scan(&queuedReady, &runningExpired, &production, &hasGatewayUser); err != nil { + return err + } + if !queuedReady && !runningExpired { + return ErrTaskExecutionLeaseUnavailable + } + if runningExpired && production { + var submissionAmbiguous bool + if err := tx.QueryRow(ctx, ` +SELECT COALESCE(( + SELECT ( + (status = 'running' AND upstream_submission_status IN ('submitting', 'response_received')) + OR (status = 'failed' AND upstream_submission_status = 'submitting') + ) + FROM gateway_task_attempts + WHERE task_id = $1::uuid + ORDER BY attempt_no DESC, started_at DESC + LIMIT 1 +), false)`, taskID).Scan(&submissionAmbiguous); err != nil { + return err + } + if submissionAmbiguous { + if _, err := tx.Exec(ctx, ` +UPDATE gateway_tasks +SET status = 'failed', + billing_status = CASE WHEN $2 THEN 'manual_review' ELSE 'not_required' END, + billing_updated_at = now(), + error = 'upstream submission result is unknown', + error_code = 'upstream_submission_unknown', + error_message = 'upstream submission result is unknown', + locked_by = NULL, + locked_at = NULL, + heartbeat_at = NULL, + execution_token = NULL, + execution_lease_expires_at = NULL, + finished_at = now(), + updated_at = now() +WHERE id = $1::uuid`, taskID, hasGatewayUser); err != nil { + return err + } + if hasGatewayUser { + payloadJSON, _ := json.Marshal(map[string]any{ + "taskId": taskID, "classification": "upstream_submission_unknown", + }) + if _, err := tx.Exec(ctx, ` +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, pricing_snapshot, payload, + status, next_attempt_at, manual_review_reason +) +SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency, + pricing_snapshot, $2::jsonb, 'manual_review', now(), 'upstream_submission_unknown' +FROM gateway_tasks +WHERE id = $1::uuid +ON CONFLICT (task_id, event_type) DO NOTHING`, taskID, string(payloadJSON)); err != nil { + return err + } + } + manualReview = true + return nil + } + } + var err error + task, err = scanGatewayTask(tx.QueryRow(ctx, ` UPDATE gateway_tasks SET status = 'running', execution_token = $2::uuid, @@ -171,10 +247,18 @@ WHERE id = $1::uuid OR (status = 'running' AND (execution_lease_expires_at IS NULL OR execution_lease_expires_at <= now())) ) RETURNING `+gatewayTaskColumns, taskID, executionToken, int(leaseTTL/time.Second))) - if errors.Is(err, pgx.ErrNoRows) { - return GatewayTask{}, ErrTaskExecutionLeaseUnavailable + return err + }) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return GatewayTask{}, ErrTaskExecutionLeaseUnavailable + } + return GatewayTask{}, err } - return task, err + if manualReview { + return GatewayTask{}, ErrTaskExecutionManualReview + } + return task, nil } func (s *Store) RenewTaskExecutionLease(ctx context.Context, taskID string, executionToken string, leaseTTL time.Duration) error { @@ -193,6 +277,14 @@ WHERE id = $1::uuid return err } if tag.RowsAffected() != 1 { + var stillRunning bool + if err := s.pool.QueryRow(ctx, ` +SELECT COALESCE((SELECT status = 'running' FROM gateway_tasks WHERE id = $1::uuid), false)`, taskID).Scan(&stillRunning); err != nil { + return err + } + if !stillRunning { + return ErrTaskExecutionFinished + } return ErrTaskExecutionLeaseLost } return nil @@ -902,8 +994,7 @@ SET status = $2, response_started_at = $6::timestamptz, response_finished_at = $7::timestamptz, response_duration_ms = $8, - finished_at = now(), - updated_at = now() + finished_at = now() WHERE id = $1::uuid`, input.AttemptID, attemptStatus, input.RequestID, input.Code, input.Message, nullableTime(input.ResponseStartedAt), nullableTime(input.ResponseFinishedAt), input.ResponseDurationMS); err != nil { return err @@ -942,7 +1033,22 @@ WHERE id = $1::uuid if tag.RowsAffected() != 1 { return ErrTaskExecutionLeaseLost } - return nil + payloadJSON, _ := json.Marshal(map[string]any{ + "taskId": input.TaskID, "classification": input.Code, + }) + _, err = tx.Exec(ctx, ` +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, pricing_snapshot, payload, + status, next_attempt_at, manual_review_reason +) +SELECT id, 'task.billing.review', 'release', reservation_amount, billing_currency, + pricing_snapshot, $2::jsonb, 'manual_review', now(), $3 +FROM gateway_tasks +WHERE id = $1::uuid + AND run_mode = 'production' + AND gateway_user_id IS NOT NULL +ON CONFLICT (task_id, event_type) DO NOTHING`, input.TaskID, string(payloadJSON), input.Code) + return err }) if err != nil { return GatewayTask{}, err diff --git a/apps/api/migrations/0069_billing_correctness_v2.sql b/apps/api/migrations/0069_billing_correctness_v2.sql index 3ababdb..659651a 100644 --- a/apps/api/migrations/0069_billing_correctness_v2.sql +++ b/apps/api/migrations/0069_billing_correctness_v2.sql @@ -95,6 +95,20 @@ ALTER TABLE model_pricing_rules UPDATE model_pricing_rules SET is_free = false WHERE is_free IS NULL; +UPDATE model_pricing_rules +SET calculator_type = CASE + WHEN resource_type IN ('text_input', 'text_cached_input', 'text_output', 'text_total') THEN 'token_usage' + WHEN resource_type = 'video' THEN 'duration_weight' + ELSE calculator_type + END, + unit = CASE + WHEN resource_type = 'video' AND unit = 'video' THEN '5s' + ELSE unit + END, + updated_at = now() +WHERE (resource_type IN ('text_input', 'text_cached_input', 'text_output', 'text_total') AND calculator_type = 'unit_weight') + OR (resource_type = 'video' AND (calculator_type = 'unit_weight' OR unit = 'video')); + ALTER TABLE model_pricing_rules ADD CONSTRAINT model_pricing_rules_explicit_free_v2_check CHECK ( base_price > 0 OR (base_price = 0 AND is_free) @@ -152,6 +166,30 @@ WHERE task.status = 'succeeded' AND transaction.transaction_type = 'task_billing' ); +INSERT INTO settlement_outbox ( + task_id, event_type, action, amount, currency, pricing_snapshot, payload, + status, next_attempt_at, manual_review_reason +) +SELECT task.id, + 'task.billing.review', + 'settle', + COALESCE(task.final_charge_amount, 0), + COALESCE(NULLIF(task.billing_summary->>'currency', ''), 'resource'), + COALESCE(task.pricing_snapshot, '{}'::jsonb), + jsonb_build_object( + 'taskId', task.id, + 'classification', 'historical_success_without_charge' + ), + 'manual_review', + now(), + 'historical_success_without_charge' +FROM gateway_tasks task +WHERE task.billing_status = 'manual_review' + AND task.status = 'succeeded' + AND task.run_mode = 'production' + AND task.gateway_user_id IS NOT NULL +ON CONFLICT (task_id, event_type) DO NOTHING; + INSERT INTO settlement_outbox ( task_id, event_type, action, amount, currency, payload, status, next_attempt_at ) diff --git a/apps/web/src/pages/PlaygroundPage.pricing.test.ts b/apps/web/src/pages/PlaygroundPage.pricing.test.ts index abacfb2..b7ff559 100644 --- a/apps/web/src/pages/PlaygroundPage.pricing.test.ts +++ b/apps/web/src/pages/PlaygroundPage.pricing.test.ts @@ -3,6 +3,7 @@ import type { GatewayPricingEstimate } from '@easyai-ai-gateway/contracts'; import { MEDIA_ESTIMATE_DEBOUNCE_MS, billingEstimateSignature, + mediaEstimateAllowsProduction, mediaEstimateFromResponse, } from './PlaygroundPage'; @@ -46,4 +47,11 @@ describe('playground pricing estimate', () => { status: 'free', }); }); + + it('only allows production submission for the current pricing signature', () => { + expect(mediaEstimateAllowsProduction({ status: 'ready', signature: 'current' }, 'current')).toBe(true); + expect(mediaEstimateAllowsProduction({ status: 'free', signature: 'current' }, 'current')).toBe(true); + expect(mediaEstimateAllowsProduction({ status: 'ready', signature: 'stale' }, 'current')).toBe(false); + expect(mediaEstimateAllowsProduction({ status: 'loading', signature: 'current' }, 'current')).toBe(false); + }); }); diff --git a/apps/web/src/pages/PlaygroundPage.tsx b/apps/web/src/pages/PlaygroundPage.tsx index 048bc6c..38daa15 100644 --- a/apps/web/src/pages/PlaygroundPage.tsx +++ b/apps/web/src/pages/PlaygroundPage.tsx @@ -66,6 +66,7 @@ type MediaEstimateState = { error?: string; pricingVersion?: string; resolver?: string; + signature?: string; status: 'idle' | 'loading' | 'ready' | 'free' | 'unavailable' | 'error'; }; @@ -192,18 +193,19 @@ export function PlaygroundPage(props: { const sequence = mediaEstimateSequenceRef.current + 1; mediaEstimateSequenceRef.current = sequence; const controller = new AbortController(); - setMediaEstimate((current) => ({ ...current, error: '', status: 'loading' })); + setMediaEstimate((current) => ({ ...current, error: '', signature: mediaEstimateSignature, status: 'loading' })); const timer = window.setTimeout(() => { estimatePricing(credential, mediaEstimatePayload, controller.signal) .then((estimate) => { if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return; - setMediaEstimate(mediaEstimateFromResponse(estimate)); + setMediaEstimate({ ...mediaEstimateFromResponse(estimate), signature: mediaEstimateSignature }); }) .catch((err) => { if (controller.signal.aborted || sequence !== mediaEstimateSequenceRef.current) return; const unavailable = err instanceof GatewayApiError && err.details.code === 'pricing_unavailable'; setMediaEstimate({ error: err instanceof Error ? err.message : unavailable ? '价格不可用' : '预计扣费计算失败', + signature: mediaEstimateSignature, status: unavailable ? 'unavailable' : 'error', }); }); @@ -326,17 +328,34 @@ export function PlaygroundPage(props: { setMediaMessage(runMode === 'video' ? '请输入视频提示词。' : '请输入图片提示词。'); return; } - if (!overrides && !mediaEstimateAllowsProduction(mediaEstimate)) { + const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads); + const runVideoMode = overrides?.videoMode ?? videoMode; + const runModelOption = modelOptions.find((item) => item.value === runModel); + const runEstimatePayload = buildMediaEstimatePayload(runMode, runModel, trimmedPrompt, runSettings, runUploads, runVideoMode, { + supportsQualityControl: runModelOption + ? deriveMediaModelCapabilities(runModelOption.models, runMode, runVideoMode, runSettings.resolution).supportsQualityControl + : mediaCapabilities?.supportsQualityControl, + }); + const runEstimateSignature = billingEstimateSignature(runEstimatePayload); + if (!overrides && !mediaEstimateAllowsProduction(mediaEstimate, runEstimateSignature)) { setMediaMessage(mediaEstimate.status === 'unavailable' ? '当前参数没有可用价格，已阻止生产生成。' : '请等待当前参数的预计费用计算完成后再生成。'); return; } + if (overrides) { + try { + await estimatePricing(credential, runEstimatePayload); + } catch (error) { + const unavailable = error instanceof GatewayApiError && error.details.code === 'pricing_unavailable'; + setMediaMessage(unavailable + ? '当前参数没有可用价格，已阻止生产生成。' + : `预计扣费计算失败，未提交生产任务：${error instanceof Error ? error.message : '未知错误'}`); + return; + } + } const localId = newLocalId(); - const runUploads = sanitizeMediaRunUploads(overrides?.uploads ?? mediaUploads); - const runVideoMode = overrides?.videoMode ?? videoMode; - const runModelOption = modelOptions.find((item) => item.value === runModel); const modelLabel = runModelOption?.label ?? runModel; const run: MediaGenerationRun = { createdAt: new Date().toISOString(), @@ -475,7 +494,7 @@ export function PlaygroundPage(props: { imageHasReference={effectiveImageHasReference} mediaSettings={mediaSettings} mediaEstimate={mediaEstimate} - submitDisabled={!mediaEstimateAllowsProduction(mediaEstimate)} + submitDisabled={!mediaEstimateAllowsProduction(mediaEstimate, mediaEstimateSignature)} mediaCapabilities={mediaCapabilities} uploadAccept={mediaUploadAcceptValue} uploadMessage={mediaUploadMessage} @@ -901,8 +920,8 @@ function mediaEstimateAriaLabel(estimate: MediaEstimateState) { return `预计扣费 ${formatEstimateAmount(estimate.amount)} ${estimate.currency || 'resource'}`; } -function mediaEstimateAllowsProduction(estimate: MediaEstimateState) { - return estimate.status === 'ready' || estimate.status === 'free'; +export function mediaEstimateAllowsProduction(estimate: MediaEstimateState, signature: string) { + return estimate.signature === signature && (estimate.status === 'ready' || estimate.status === 'free'); } export function billingEstimateSignature(payload: Record | null) { diff --git a/apps/web/src/pages/admin/BillingSettlementsPanel.tsx b/apps/web/src/pages/admin/BillingSettlementsPanel.tsx index 00cac21..17f5829 100644 --- a/apps/web/src/pages/admin/BillingSettlementsPanel.tsx +++ b/apps/web/src/pages/admin/BillingSettlementsPanel.tsx @@ -2,13 +2,14 @@ import { useCallback, useEffect, useState } from 'react'; import { AlertTriangle, ReceiptText, RefreshCw, RotateCcw } from 'lucide-react'; import type { BillingSettlement } from '@easyai-ai-gateway/contracts'; import { listBillingSettlements, retryBillingSettlement } from '../../api'; -import { Badge, Button, Card, CardContent, CardHeader, CardTitle, Select, Table, TableCell, TableHead, TableRow } from '../../components/ui'; +import { Badge, Button, Card, CardContent, CardHeader, CardTitle, ConfirmDialog, Select, Table, TableCell, TableHead, TableRow } from '../../components/ui'; export function BillingSettlementsPanel(props: { token: string }) { const [items, setItems] = useState([]); const [status, setStatus] = useState(''); const [loading, setLoading] = useState(false); const [retrying, setRetrying] = useState(''); + const [confirming, setConfirming] = useState(null); const [message, setMessage] = useState(''); const load = useCallback(async (signal?: AbortSignal) => { @@ -95,8 +96,8 @@ export function BillingSettlementsPanel(props: { token: string }) { {item.lastErrorCode || item.manualReviewReason || '-'} {(item.status === 'retryable_failed' || item.status === 'manual_review') ? ( - ) : '-'} @@ -106,6 +107,19 @@ export function BillingSettlementsPanel(props: { token: string }) { ) : ( {loading ? '正在加载结算队列' : '暂无结算记录'}可切换状态筛选，查看待处理与人工复核记录。 )} + setConfirming(null)} + onConfirm={() => { + if (!confirming) return; + const item = confirming; + void retry(item).finally(() => setConfirming(null)); + }} + /> ); } @@ -126,6 +140,23 @@ function formatAmount(value: number) { return new Intl.NumberFormat('zh-CN', { minimumFractionDigits: 0, maximumFractionDigits: 9 }).format(value); } +function retryActionLabel(item: BillingSettlement) { + if (item.status !== 'manual_review') return '重试'; + if (item.manualReviewReason === 'upstream_submission_unknown' && item.action === 'release') return '确认释放'; + if (item.manualReviewReason === 'historical_success_without_charge' && item.action === 'settle') return '确认结算'; + return '重新入队'; +} + +function retryActionDescription(item: BillingSettlement) { + if (item.manualReviewReason === 'upstream_submission_unknown' && item.action === 'release') { + return '上游提交结果不明。确认后只会释放当前冻结，不会再次调用上游；请先完成外部核查。'; + } + if (item.manualReviewReason === 'historical_success_without_charge' && item.action === 'settle') { + return `确认后将按历史快照结算 ${formatAmount(item.amount)} ${item.currency}，不会重新调用上游。`; + } + return '确认后会将该记录重新放入结算队列，不会重新调用上游。'; +} + function shortId(value: string) { return value.length > 12 ? `${value.slice(0, 8)}…${value.slice(-4)}` : value; } diff --git a/docs/billing-flow-v2.md b/docs/billing-flow-v2.md index 4944897..e122344 100644 --- a/docs/billing-flow-v2.md +++ b/docs/billing-flow-v2.md @@ -26,7 +26,9 @@ sequenceDiagram ## 失败与取消 -上游提交前失败、上游明确失败和用户取消均在任务终态事务内写入 `release` Outbox。Worker 使用冻结流水的幂等键释放资金。若上游请求已经进入 `submitting` 但没有可靠响应，系统不得再次提交；任务转人工复核并保留冻结。 +上游提交前失败、上游明确失败和用户取消均在任务终态事务内写入 `release` Outbox。Worker 使用冻结流水的幂等键释放资金。若上游请求已经进入 `submitting`，或成功响应已返回但任务终态尚未提交，租约接管必须停止自动执行；任务和对应 Outbox 转人工复核并保留冻结。 + +人工复核记录在管理端明确显示后续动作。上游结果不明的记录只能由 Manager 在完成外部核查后确认释放；历史成功未扣费且金额快照可用的记录可由 Manager 明确确认结算。两类操作均要求确认、`Idempotency-Key` 和审计记录，且不会再次调用上游。 ## 即时估价 -- 2.54.0 From b7bb9ed8d5b781c0efd0f49eb8a25e1a7c7dbe5a Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 10:33:39 +0800 Subject: [PATCH 11/13] =?UTF-8?q?fix(ci):=20=E4=B8=BA=E9=9B=86=E6=88=90?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=8F=90=E4=BE=9B=E5=9B=BA=E5=AE=9A=20Docker?= =?UTF-8?q?=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 从固定 digest 的 Runner 镜像提取 Docker CLI 29.6.0，校验二进制 SHA-256 后通过只读工具链挂载给 Job。 同步 PR 与 Tag 工作流的版本探针，并增加流水线契约测试，避免 PostgreSQL 集成测试再次因缺少 docker 命令失败。 --- .gitea/workflows/ci.yml | 1 + .gitea/workflows/release-ci.yml | 1 + docs/runbooks/production-ci-cd.md | 2 +- scripts/provision-ci-runner.sh | 18 ++++++++++++++++-- tests/ci/pipeline-test.sh | 7 +++++++ 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 5978943..ff74a71 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: go version node --version pnpm --version + docker --version docker-compose version shellcheck --version trivy --version diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml index 80b389b..a5dbdef 100644 --- a/.gitea/workflows/release-ci.yml +++ b/.gitea/workflows/release-ci.yml @@ -31,6 +31,7 @@ jobs: go version node --version pnpm --version + docker --version docker-compose version shellcheck --version trivy --version diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index f3e7a9c..ad30679 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -16,7 +16,7 @@ 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 -provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。 +provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0`、Docker Compose `5.3.1`，并从固定 digest 的 Runner 镜像提取 Docker CLI `29.6.0` 后校验二进制 SHA-256；不会依赖宿主是否预装这些命令。所有工具都在精确 Node Job 镜像内实际执行版本检查。 ## 首次安装或修复 CI Runner diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh index 6426a09..8cf4848 100755 --- a/scripts/provision-ci-runner.sh +++ b/scripts/provision-ci-runner.sh @@ -26,6 +26,8 @@ SHELLCHECK_VERSION=0.11.0 SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 COMPOSE_VERSION=5.3.1 COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 +DOCKER_CLI_VERSION=29.6.0 +DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba PNPM_VERSION=10.18.1 RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' @@ -188,14 +190,25 @@ tar -xzf "$PREFIX/downloads/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" -C "$PREF chmod 0755 "$PREFIX/bin/trivy" GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 -chown -R root:root "$PREFIX" -chmod -R go-w "$PREFIX" require_free_space "$MIN_FREE_GIB" "runner image pull" docker pull "$RUNNER_IMAGE_SOURCE" runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ fail "pinned runner image did not resolve to an immutable local image ID" + +# The job image intentionally contains no Docker client. Extract the static CLI +# from the immutable Runner image, then verify its binary digest before exposing +# it through the read-only CI toolchain mount. +docker_cli_download="$PREFIX/downloads/docker-${DOCKER_CLI_VERSION}-linux-x86_64" +docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image" \ + /usr/local/bin/docker > "$docker_cli_download" +printf '%s %s\n' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c - +install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker" +"$PREFIX/bin/docker" --version | grep -Fq "Docker version ${DOCKER_CLI_VERSION}," +chown -R root:root "$PREFIX" +chmod -R go-w "$PREFIX" + require_free_space "$MIN_POST_INSTALL_FREE_GIB" "completed runner image pull" docker volume create "$RUNNER_VOLUME" >/dev/null @@ -298,6 +311,7 @@ docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \ go version node --version pnpm --version + docker --version docker-compose version shellcheck --version trivy --version diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index bfdedbf..a6edbe1 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -116,6 +116,7 @@ for quality_workflow in "$workflow" "$release_workflow"; do grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow" + grep -Fq 'docker --version' "$quality_workflow" grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" if grep -Fq 'docker compose' "$quality_workflow"; then echo 'job container must use the read-only standalone Compose binary' >&2 @@ -264,6 +265,8 @@ grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" +grep -q '^DOCKER_CLI_VERSION=29.6.0$' "$provision" +grep -q '^DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba$' "$provision" grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" @@ -293,6 +296,10 @@ grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" grep -Fq 'docker-compose-linux-x86_64' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" +grep -Fq 'docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image"' "$provision" +grep -Fq 'printf '\''%s %s\n'\'' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c -' "$provision" +grep -Fq 'install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker"' "$provision" +grep -Fq '"$PREFIX/bin/docker" --version' "$provision" grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision" -- 2.54.0 From 3561efa7da022982ca7c2c7f1b1312b08afc091c Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 10:36:56 +0800 Subject: [PATCH 12/13] =?UTF-8?q?fix(ci):=20=E5=85=81=E8=AE=B8=E5=A4=8D?= =?UTF-8?q?=E7=94=A8=E5=9B=BA=E5=AE=9A=E6=91=98=E8=A6=81=E7=9A=84=20Runner?= =?UTF-8?q?=20=E9=95=9C=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 本地已存在完全相同 digest 的 Runner 镜像时跳过外部拉取，继续执行二进制校验与隔离探针；仅在镜像缺失时访问仓库。 避免 Docker Hub 瞬时不可达导致专用 CI Runner 无法恢复。 --- docs/runbooks/production-ci-cd.md | 2 +- scripts/provision-ci-runner.sh | 4 +++- tests/ci/pipeline-test.sh | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index ad30679..794170f 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -16,7 +16,7 @@ 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 -provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0`、Docker Compose `5.3.1`，并从固定 digest 的 Runner 镜像提取 Docker CLI `29.6.0` 后校验二进制 SHA-256；不会依赖宿主是否预装这些命令。所有工具都在精确 Node Job 镜像内实际执行版本检查。 +provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0`、Docker Compose `5.3.1`，并从固定 digest 的 Runner 镜像提取 Docker CLI `29.6.0` 后校验二进制 SHA-256；不会依赖宿主是否预装这些命令。固定 digest 的 Runner 镜像已存在时可离线复用，缺失时才访问镜像仓库。所有工具都在精确 Node Job 镜像内实际执行版本检查。 ## 首次安装或修复 CI Runner diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh index 8cf4848..eae66c7 100755 --- a/scripts/provision-ci-runner.sh +++ b/scripts/provision-ci-runner.sh @@ -192,7 +192,9 @@ GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 require_free_space "$MIN_FREE_GIB" "runner image pull" -docker pull "$RUNNER_IMAGE_SOURCE" +if ! docker image inspect "$RUNNER_IMAGE_SOURCE" >/dev/null 2>&1; then + docker pull "$RUNNER_IMAGE_SOURCE" +fi runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ fail "pinned runner image did not resolve to an immutable local image ID" diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index a6edbe1..7a609fb 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -272,6 +272,7 @@ grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision" grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision" +grep -Fq 'if ! docker image inspect "$RUNNER_IMAGE_SOURCE"' "$provision" grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision" grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision" grep -Fq "JOB_IMAGE='$job_image'" "$provision" -- 2.54.0 From 0aa9b3e88f1341616751994f3e403f065f58f08d Mon Sep 17 00:00:00 2001 From: chengcheng Date: Tue, 21 Jul 2026 10:45:52 +0800 Subject: [PATCH 13/13] =?UTF-8?q?fix(ci):=20=E4=BD=BF=E7=94=A8=E9=9A=94?= =?UTF-8?q?=E7=A6=BB=E6=9C=8D=E5=8A=A1=E8=BF=90=E8=A1=8C=20PostgreSQL=20?= =?UTF-8?q?=E9=9B=86=E6=88=90=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 PostgreSQL 16 改为固定 digest 的 Actions service container，并通过专用 Job 网络向 Go migrator 和集成测试提供数据库。 移除 PR Job 对 Docker CLI 的依赖，继续禁止宿主及内层 Docker API 暴露；同步发布工作流、契约测试和镜像代理恢复说明。 --- .gitea/workflows/ci.yml | 55 ++++++++++--------------------- .gitea/workflows/release-ci.yml | 55 ++++++++++--------------------- docs/runbooks/production-ci-cd.md | 10 +++++- scripts/provision-ci-runner.sh | 17 +--------- tests/ci/pipeline-test.sh | 13 +++----- 5 files changed, 49 insertions(+), 101 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index ff74a71..58f1805 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -9,8 +9,22 @@ on: jobs: verify: runs-on: easyai-gateway-ci-unprivileged-v2 + services: + postgres: + image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_USER: easyai_test + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: easyai_gateway_test + options: >- + --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" + --health-interval 2s + --health-timeout 5s + --health-retries 30 env: TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 + AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable + AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable steps: - name: Checkout without external Actions env: @@ -47,7 +61,6 @@ jobs: go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version @@ -80,37 +93,9 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } - - name: Start PostgreSQL 16 integration database - env: - CI_RUN_ID: ${{ github.run_id }} - CI_RUN_NUMBER: ${{ github.run_number }} - run: | - set -eu - container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" - docker run -d --rm --name "$container" \ - -e POSTGRES_USER=easyai_test \ - -e POSTGRES_PASSWORD=easyai_test_only \ - -e POSTGRES_DB=easyai_gateway_test \ - postgres:16-alpine - for attempt in $(seq 1 60); do - if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then - break - fi - test "$attempt" -lt 60 - sleep 1 - done - database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") - test -n "$database_ip" - printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" - printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' - for migration in apps/api/migrations/*.sql; do - docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" - version=$(basename "$migration" .sql) - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" - done + - name: Migrate PostgreSQL 16 integration database + working-directory: apps/api + run: go run ./cmd/migrate - name: Verify Go code working-directory: apps/api env: @@ -142,9 +127,3 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . - - name: Stop PostgreSQL integration database - if: always() - run: | - if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then - docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true - fi diff --git a/.gitea/workflows/release-ci.yml b/.gitea/workflows/release-ci.yml index a5dbdef..7142922 100644 --- a/.gitea/workflows/release-ci.yml +++ b/.gitea/workflows/release-ci.yml @@ -7,8 +7,22 @@ on: jobs: verify-tag: runs-on: easyai-gateway-ci-unprivileged-v2 + services: + postgres: + image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_USER: easyai_test + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: easyai_gateway_test + options: >- + --health-cmd "pg_isready -U easyai_test -d easyai_gateway_test" + --health-interval 2s + --health-timeout 5s + --health-retries 30 env: TRIVY_DB_REPOSITORY: ghcr.m.daocloud.io/aquasecurity/trivy-db:2 + AI_GATEWAY_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable + AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable steps: - name: Checkout without external Actions env: @@ -31,7 +45,6 @@ jobs: go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version @@ -65,37 +78,9 @@ jobs: printf 'Go files require gofmt:\n%s\n' "$unformatted" >&2 exit 1 } - - name: Start PostgreSQL 16 integration database - env: - CI_RUN_ID: ${{ github.run_id }} - CI_RUN_NUMBER: ${{ github.run_number }} - run: | - set -eu - container="easyai-gateway-test-pg-${CI_RUN_ID}-${CI_RUN_NUMBER}" - docker run -d --rm --name "$container" \ - -e POSTGRES_USER=easyai_test \ - -e POSTGRES_PASSWORD=easyai_test_only \ - -e POSTGRES_DB=easyai_gateway_test \ - postgres:16-alpine - for attempt in $(seq 1 60); do - if docker exec "$container" pg_isready -U easyai_test -d easyai_gateway_test >/dev/null 2>&1; then - break - fi - test "$attempt" -lt 60 - sleep 1 - done - database_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container") - test -n "$database_ip" - printf 'AI_GATEWAY_TEST_DATABASE_URL=postgresql://easyai_test:easyai_test_only@%s:5432/easyai_gateway_test?sslmode=disable\n' "$database_ip" >> "$GITHUB_ENV" - printf 'AI_GATEWAY_TEST_POSTGRES_CONTAINER=%s\n' "$container" >> "$GITHUB_ENV" - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c 'CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())' - for migration in apps/api/migrations/*.sql; do - docker exec -i "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test < "$migration" - version=$(basename "$migration" .sql) - docker exec "$container" psql -v ON_ERROR_STOP=1 -U easyai_test -d easyai_gateway_test \ - -c "INSERT INTO schema_migrations(version) VALUES ('$version') ON CONFLICT (version) DO NOTHING" - done + - name: Migrate PostgreSQL 16 integration database + working-directory: apps/api + run: go run ./cmd/migrate - name: Verify Go code working-directory: apps/api env: @@ -127,9 +112,3 @@ jobs: trivy fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL \ --ignore-unfixed --exit-code 1 --timeout 15m --skip-dirs .git \ --skip-dirs node_modules . - - name: Stop PostgreSQL integration database - if: always() - run: | - if test -n "${AI_GATEWAY_TEST_POSTGRES_CONTAINER:-}"; then - docker rm -f "$AI_GATEWAY_TEST_POSTGRES_CONTAINER" >/dev/null 2>&1 || true - fi diff --git a/docs/runbooks/production-ci-cd.md b/docs/runbooks/production-ci-cd.md index 794170f..1fbd25c 100644 --- a/docs/runbooks/production-ci-cd.md +++ b/docs/runbooks/production-ci-cd.md @@ -16,7 +16,7 @@ 生产 CD 由部署仓的 root-owned dispatcher 执行。它在相同源码 SHA 上同时等待 `ci / verify (push)` 与本次新产生的 `release-ci / verify-tag (push)` 成功，然后只运行部署仓中固定的 BuildKit 构建、Trivy 镜像扫描、digest 发布、备份、迁移、健康检查和回滚命令；不得执行 Tag checkout 中的 `pnpm`、测试、shell 脚本或应用程序。 -provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0`、Docker Compose `5.3.1`，并从固定 digest 的 Runner 镜像提取 Docker CLI `29.6.0` 后校验二进制 SHA-256；不会依赖宿主是否预装这些命令。固定 digest 的 Runner 镜像已存在时可离线复用，缺失时才访问镜像仓库。所有工具都在精确 Node Job 镜像内实际执行版本检查。 +provision 安装 checksum-pinned 的官方静态 ShellCheck `0.11.0` 和 Docker Compose `5.3.1`，不会依赖宿主是否预装这些命令。所有下载都在使用前校验固定 SHA-256，并在精确 Node Job 镜像内实际执行版本检查。PostgreSQL 16 集成库作为固定 digest 的 Actions service container 运行，PR Job 不接收宿主或内层 Docker API。 ## 首次安装或修复 CI Runner @@ -34,6 +34,14 @@ RUNNER_REGISTRATION_TOKEN='<一次性 Token>' ./scripts/provision-ci-runner.sh ./scripts/provision-ci-runner.sh ``` +宿主无法直连 Docker Hub 时，可使用可信 HTTPS 镜像代理；Runner 来源仍必须携带脚本中完全相同的 manifest digest，内层 daemon 拉取的 Job 与 service 镜像也会继续校验工作流固定的 digest： + +```bash +CI_RUNNER_IMAGE_SOURCE='docker.m.daocloud.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' \ +CI_NESTED_DOCKER_REGISTRY_MIRROR='https://docker.m.daocloud.io' \ +./scripts/provision-ci-runner.sh +``` + 注册 Token 通过交互 stdin 送给短命注册容器，不出现在宿主或容器的命令行参数中；provision 随后立即从环境中清除它，长期 systemd unit 也不包含 Token。不要输出或复制 `/data/.runner` 内容。脚本会停用旧 host Runner，并确保旧 Runner 用户和 v2 遗留用户不属于 `docker`/`sudo` 组。 默认磁盘门禁可用环境变量提高，但生产环境不应降低： diff --git a/scripts/provision-ci-runner.sh b/scripts/provision-ci-runner.sh index eae66c7..91b8db7 100755 --- a/scripts/provision-ci-runner.sh +++ b/scripts/provision-ci-runner.sh @@ -26,8 +26,6 @@ SHELLCHECK_VERSION=0.11.0 SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198 COMPOSE_VERSION=5.3.1 COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959 -DOCKER_CLI_VERSION=29.6.0 -DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba PNPM_VERSION=10.18.1 RUNNER_IMAGE='docker.io/gitea/runner:2.0.0-dind-rootless@sha256:5b7b625ff773d0ee761788c47582503ec1b241fa5b81edebad48a57e663f4f3a' JOB_IMAGE='docker.io/library/node:24.16.0-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7' @@ -192,22 +190,10 @@ GOBIN="$PREFIX/bin" GOPROXY="${GO_MODULE_PROXY:-https://goproxy.cn,direct}" \ "$PREFIX/toolchains/go/bin/go" install golang.org/x/vuln/cmd/govulncheck@v1.6.0 require_free_space "$MIN_FREE_GIB" "runner image pull" -if ! docker image inspect "$RUNNER_IMAGE_SOURCE" >/dev/null 2>&1; then - docker pull "$RUNNER_IMAGE_SOURCE" -fi +docker pull "$RUNNER_IMAGE_SOURCE" runner_runtime_image=$(docker image inspect --format '{{.Id}}' "$RUNNER_IMAGE_SOURCE") printf '%s\n' "$runner_runtime_image" | grep -Eq '^sha256:[0-9a-f]{64}$' || \ fail "pinned runner image did not resolve to an immutable local image ID" - -# The job image intentionally contains no Docker client. Extract the static CLI -# from the immutable Runner image, then verify its binary digest before exposing -# it through the read-only CI toolchain mount. -docker_cli_download="$PREFIX/downloads/docker-${DOCKER_CLI_VERSION}-linux-x86_64" -docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image" \ - /usr/local/bin/docker > "$docker_cli_download" -printf '%s %s\n' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c - -install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker" -"$PREFIX/bin/docker" --version | grep -Fq "Docker version ${DOCKER_CLI_VERSION}," chown -R root:root "$PREFIX" chmod -R go-w "$PREFIX" @@ -313,7 +299,6 @@ docker exec "$PROBE_CONTAINER" docker run --rm --pull=never \ go version node --version pnpm --version - docker --version docker-compose version shellcheck --version trivy --version diff --git a/tests/ci/pipeline-test.sh b/tests/ci/pipeline-test.sh index 7a609fb..7ff545d 100755 --- a/tests/ci/pipeline-test.sh +++ b/tests/ci/pipeline-test.sh @@ -116,8 +116,12 @@ for quality_workflow in "$workflow" "$release_workflow"; do grep -Fq './tests/ci/pipeline-test.sh' "$quality_workflow" grep -Fq './tests/ci/semver-test.sh' "$quality_workflow" grep -Fq 'docker-compose version' "$quality_workflow" - grep -Fq 'docker --version' "$quality_workflow" grep -Fq 'docker-compose -f docker-compose.yml config --quiet' "$quality_workflow" + grep -Fq 'services:' "$quality_workflow" + grep -Fq 'image: docker.io/library/postgres:16-alpine@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777' "$quality_workflow" + grep -Fq 'POSTGRES_HOST_AUTH_METHOD: trust' "$quality_workflow" + grep -Fq 'AI_GATEWAY_TEST_DATABASE_URL: postgresql://easyai_test@postgres:5432/easyai_gateway_test?sslmode=disable' "$quality_workflow" + grep -Fq 'run: go run ./cmd/migrate' "$quality_workflow" if grep -Fq 'docker compose' "$quality_workflow"; then echo 'job container must use the read-only standalone Compose binary' >&2 exit 1 @@ -265,14 +269,11 @@ grep -q '^SHELLCHECK_VERSION=0.11.0$' "$provision" grep -q '^SHELLCHECK_SHA256=8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198$' "$provision" grep -q '^COMPOSE_VERSION=5.3.1$' "$provision" grep -q '^COMPOSE_SHA256=f9ebc6ebdb19d769b793c245a736caaeb198c62587f13b25c660c13b4987f959$' "$provision" -grep -q '^DOCKER_CLI_VERSION=29.6.0$' "$provision" -grep -q '^DOCKER_CLI_SHA256=80b40fa11d797a71ea15f9a31e7004790ddac95f58a8222a40c08f08a639faba$' "$provision" grep -Fq 'RUNNER_NAME=${RUNNER_NAME:-easyai-gateway-ci-unprivileged-v2}' "$provision" grep -Fq "RUNNER_IMAGE='$runner_image'" "$provision" grep -Fq 'RUNNER_IMAGE_SOURCE=${CI_RUNNER_IMAGE_SOURCE:-$RUNNER_IMAGE}' "$provision" grep -Fq 'runner_manifest_digest=${RUNNER_IMAGE##*@}' "$provision" grep -Fq 'CI_RUNNER_IMAGE_SOURCE must use the pinned runner manifest digest' "$provision" -grep -Fq 'if ! docker image inspect "$RUNNER_IMAGE_SOURCE"' "$provision" grep -Fq 'runner_runtime_image=$(docker image inspect' "$provision" grep -Fq 'RUNNER_IMAGE_REF="%s"' "$provision" grep -Fq "JOB_IMAGE='$job_image'" "$provision" @@ -297,10 +298,6 @@ grep -Fq 'shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz' "$provision" grep -Fq 'docker-compose-linux-x86_64' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "$PREFIX/bin/shellcheck"' "$provision" grep -Fq 'install -m 0755 "$PREFIX/downloads/docker-compose-${COMPOSE_VERSION}-linux-x86_64" "$PREFIX/bin/docker-compose"' "$provision" -grep -Fq 'docker run --rm --pull=never --entrypoint /bin/cat "$runner_runtime_image"' "$provision" -grep -Fq 'printf '\''%s %s\n'\'' "$DOCKER_CLI_SHA256" "$docker_cli_download" | sha256sum -c -' "$provision" -grep -Fq 'install -m 0755 "$docker_cli_download" "$PREFIX/bin/docker"' "$provision" -grep -Fq '"$PREFIX/bin/docker" --version' "$provision" grep -Fq 'PATH="$PREFIX/toolchains/node/bin:$PATH"' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 docker' "$provision" grep -Fq 'gpasswd -d easyai-gateway-ci-v2 sudo' "$provision" -- 2.54.0