diff --git a/docs/operations/production-acceptance.md b/docs/operations/production-acceptance.md index 0a4451b..bbdd4ec 100644 --- a/docs/operations/production-acceptance.md +++ b/docs/operations/production-acceptance.md @@ -132,6 +132,16 @@ scripts/cluster/run-production-acceptance.sh \ --local-report dist/acceptance/local/<本地run-id>/acceptance-report.json ``` +只有用户明确要求跳过本地验收时,才允许使用下面的审计豁免。报告会把 `localNative` 和 +`amd64Artifact` 记录为 `skipped`,并保留 `local_acceptance_explicitly_waived` 记录;不得 +伪造为通过。线上模拟、真实金丝雀、资源、一致性和发布 CAS 门禁仍全部执行: + +```bash +scripts/cluster/run-production-acceptance.sh \ + --execute dist/releases/<完整SHA>.json \ + --skip-local-acceptance +``` + 脚本依次执行: 1. 校验工作区、manifest、线上 release SHA、digest、双站点镜像和副本数。 diff --git a/scripts/acceptance/report.mjs b/scripts/acceptance/report.mjs index cd48e0b..84b032b 100755 --- a/scripts/acceptance/report.mjs +++ b/scripts/acceptance/report.mjs @@ -86,13 +86,66 @@ function validate(report) { throw new Error('acceptance report stages, gates, or promotion are invalid'); } for (const [name, stage] of Object.entries(report.stages)) { - if (!['passed', 'failed', 'not-run'].includes(stage?.status)) { + if (!['passed', 'failed', 'not-run', 'skipped'].includes(stage?.status)) { throw new Error(`invalid stage status for ${name}`); } } assertSecretSafe(report); } +async function buildOnlineWaiver(values) { + for (const key of ['release-sha', 'api-digest', 'snapshot', 'waiver-reason', 'output']) { + if (!values[key]) throw new Error(`--${key} is required`); + } + const snapshot = await regularJSON(values.snapshot); + if (!shaPattern.test(values['release-sha']) || !digestPattern.test(values['api-digest']) || + snapshot.schemaVersion !== 'acceptance-snapshot/v1' || + !shaPattern.test(snapshot.source?.releaseSha ?? '') || + !hashPattern.test(snapshot.source?.configHash ?? '') || + !hashPattern.test(snapshot.snapshotSha256 ?? '') || + !/^[a-z0-9_]{3,80}$/.test(values['waiver-reason'])) { + throw new Error('online waiver identity fields or reason are invalid'); + } + const report = { + schemaVersion: 'acceptance-report/v1', + runId: `online-waiver-${values['release-sha'].slice(0, 12)}`, + release: { + sourceSha: values['release-sha'], + apiImageDigest: values['api-digest'], + workerImageDigest: values['api-digest'] + }, + snapshot: { + sourceReleaseSha: snapshot.source.releaseSha, + configHash: snapshot.source.configHash, + sha256: snapshot.snapshotSha256 + }, + stages: { + localNative: { status: 'skipped', waiverReason: values['waiver-reason'] }, + amd64Artifact: { status: 'skipped', waiverReason: values['waiver-reason'] }, + onlineSimulation: { status: 'not-run' }, + realCanary: { status: 'not-run' } + }, + certifiedProfile: null, + gates: [ + { + id: 'local_acceptance_explicitly_waived', + passed: true, + detail: values['waiver-reason'] + } + ], + promotion: { + ready: false, + requiresManualConfirmation: true, + trafficMode: 'not-applicable' + }, + createdAt: new Date().toISOString(), + secretSafe: true + }; + validate(report); + await writeNewJSON(values.output, report); + process.stdout.write('acceptance_report_online_waiver=PASS\n'); +} + async function loadReports(directory) { const entries = (await readdir(resolve(directory), { withFileTypes: true })) .filter((entry) => entry.isFile() && entry.name.endsWith('.json')) @@ -260,12 +313,13 @@ async function mergeProduction(values) { const local = await regularJSON(values['local-report']); const production = await regularJSON(values['production-summary']); validate(local); - if (local.release.sourceSha !== production.releaseSha || + if (!production.runId || local.release.sourceSha !== production.releaseSha || local.release.apiImageDigest !== production.apiImageDigest || local.snapshot.configHash !== production.snapshotConfigHash || local.snapshot.sha256 !== production.snapshotSha256) { throw new Error('production summary does not match local acceptance CAS fields'); } + local.runId = production.runId; local.stages.onlineSimulation = { status: production.passed === true ? 'passed' : 'failed', stableCapacityProfile: production.stableCapacityProfile, @@ -299,6 +353,8 @@ if (command === 'validate') { await buildLocal(values); } else if (command === 'build-local-partial') { await buildLocalPartial(values); +} else if (command === 'build-online-waiver') { + await buildOnlineWaiver(values); } else if (command === 'merge-production') { await mergeProduction(values); } else if (command === 'mark-promoted') { @@ -338,5 +394,5 @@ if (command === 'validate') { await replaceJSONAtomically(input, output, report); process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`); } else { - throw new Error('usage: report.mjs {validate|build-local|build-local-partial|merge-production|mark-promoted|attach-monitor} [options]'); + throw new Error('usage: report.mjs {validate|build-local|build-local-partial|build-online-waiver|merge-production|mark-promoted|attach-monitor} [options]'); } diff --git a/scripts/cluster/run-production-acceptance.sh b/scripts/cluster/run-production-acceptance.sh index 351d0b9..75f2e73 100755 --- a/scripts/cluster/run-production-acceptance.sh +++ b/scripts/cluster/run-production-acceptance.sh @@ -12,6 +12,9 @@ Usage: scripts/cluster/run-production-acceptance.sh \ --execute dist/releases/.json \ --local-report dist/acceptance/local//acceptance-report.json + scripts/cluster/run-production-acceptance.sh \ + --execute dist/releases/.json \ + --skip-local-acceptance scripts/cluster/run-production-acceptance.sh \ --promote dist/releases/.json --run-id @@ -22,6 +25,9 @@ access to the selected production candidates. A failed run deliberately keeps traffic in validation mode and restores only the last stable capacity profile. Successful execute runs also remain in validation. Only the separate --promote action performs the final CAS switch to live after manual confirmation. +The explicit --skip-local-acceptance form records both local stages as skipped +with a user-directed waiver. It never records them as passed and preserves all +online simulation, real canary, resource, consistency, and release CAS gates. Required private environment values: AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN @@ -44,21 +50,28 @@ if [[ ${1:-} == --promote ]]; then exec "$script_dir/promote-production-acceptance.sh" "$@" fi -[[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]] || { +skip_local_acceptance=false +local_acceptance_report= +if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then + release_manifest=$2 + local_acceptance_report=$4 +elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then + release_manifest=$2 + skip_local_acceptance=true +else usage >&2 exit 64 -} - -release_manifest=$2 -local_acceptance_report=$4 +fi [[ -f $release_manifest && ! -L $release_manifest ]] || { echo 'release manifest must be a regular file' >&2 exit 1 } -[[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || { - echo 'local acceptance report must be a regular file' >&2 - exit 1 -} +if [[ $skip_local_acceptance != true ]]; then + [[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || { + echo 'local acceptance report must be a regular file' >&2 + exit 1 + } +fi load_cluster_env require_commands curl git go jq node openssl sed @@ -136,17 +149,19 @@ api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_mani } api_digest=${api_image##*@} worker_digest=$api_digest -node "$cluster_root/scripts/acceptance/report.mjs" validate \ - --input "$local_acceptance_report" >/dev/null -[[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" && - $(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" && - $(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed && - $(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed && - $(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run && - $(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || { - echo 'local acceptance report does not match this release or has invalid stage state' >&2 - exit 1 -} +if [[ $skip_local_acceptance != true ]]; then + node "$cluster_root/scripts/acceptance/report.mjs" validate \ + --input "$local_acceptance_report" >/dev/null + [[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" && + $(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" && + $(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed && + $(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed && + $(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run && + $(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || { + echo 'local acceptance report does not match this release or has invalid stage state' >&2 + exit 1 + } +fi [[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" ]] || { echo 'working copy HEAD must equal the release SHA' >&2 exit 1 @@ -185,8 +200,12 @@ AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot current_production_snapshot=$temporary_root/current-production-snapshot.json -local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report") -local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report") +local_snapshot_config_hash= +local_snapshot_sha256= +if [[ $skip_local_acceptance != true ]]; then + local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report") + local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report") +fi active_load_pid= active_pressure_pid= capacity_profiles_payload= @@ -232,10 +251,22 @@ AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \ "$acceptance_snapshot_binary" export \ --release-sha "$release_sha" \ --output "$current_production_snapshot" >/dev/null -[[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || { - echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2 - exit 1 -} +if [[ $skip_local_acceptance == true ]]; then + local_acceptance_report=$temporary_root/online-waiver-base.json + node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \ + --release-sha "$release_sha" \ + --api-digest "$api_digest" \ + --snapshot "$current_production_snapshot" \ + --waiver-reason user_directed_online_only_acceptance \ + --output "$local_acceptance_report" >/dev/null + local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report") + local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report") +else + [[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || { + echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2 + exit 1 + } +fi remote_kubectl() { local command_text @@ -1073,6 +1104,7 @@ mark_run_failed() { } }') jq -n \ + --arg runId "$run_id" \ --arg releaseSha "$release_sha" \ --arg apiImageDigest "$api_digest" \ --arg snapshotConfigHash "$local_snapshot_config_hash" \ @@ -1082,6 +1114,7 @@ mark_run_failed() { --arg reason "$reason" \ --argjson artifacts "$artifact_names" \ '{ + runId:$runId, releaseSha:$releaseSha, apiImageDigest:$apiImageDigest, snapshotConfigHash:$snapshotConfigHash, diff --git a/tests/acceptance/report-test.mjs b/tests/acceptance/report-test.mjs index c67dc38..0f8bc1e 100644 --- a/tests/acceptance/report-test.mjs +++ b/tests/acceptance/report-test.mjs @@ -22,6 +22,7 @@ const output = join(temporary, 'acceptance-report.json'); const partialOutput = join(temporary, 'acceptance-report.partial.json'); const productionPartial = join(temporary, 'production-summary.partial.json'); const mergedPartialOutput = join(temporary, 'acceptance-report.production-partial.json'); +const waivedOutput = join(temporary, 'acceptance-report.waived.json'); writeJSON(runtime, { schemaVersion: 'acceptance-runtime/v1', runId: runID, @@ -108,6 +109,7 @@ assert.equal(readFileSync(partialOutput, 'utf8').includes('must-not-be-copied'), execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', partialOutput]); writeJSON(productionPartial, { + runId: runID, releaseSha, apiImageDigest: digest, snapshotConfigHash: configHash, @@ -132,8 +134,25 @@ execFileSync('node', [ ]); const mergedPartial = JSON.parse(readFileSync(mergedPartialOutput, 'utf8')); assert.equal(mergedPartial.stages.onlineSimulation.status, 'failed'); +assert.equal(mergedPartial.runId, runID); assert.equal(mergedPartial.stages.realCanary.status, 'failed'); assert.equal(mergedPartial.promotion.ready, false); assert.equal(mergedPartial.promotion.trafficMode, 'validation'); assert.equal(mergedPartial.gates.at(-1).id, 'workflow_interrupted'); execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', mergedPartialOutput]); + +execFileSync('node', [ + join(root, 'scripts/acceptance/report.mjs'), + 'build-online-waiver', + '--release-sha', releaseSha, + '--api-digest', digest, + '--snapshot', snapshot, + '--waiver-reason', 'user_directed_online_only_acceptance', + '--output', waivedOutput +]); +const waived = JSON.parse(readFileSync(waivedOutput, 'utf8')); +assert.equal(waived.stages.localNative.status, 'skipped'); +assert.equal(waived.stages.amd64Artifact.status, 'skipped'); +assert.equal(waived.gates[0].id, 'local_acceptance_explicitly_waived'); +assert.equal(waived.promotion.ready, false); +execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', waivedOutput]);