fix(acceptance): 支持显式跳过本地验收

在用户明确授权时,将本地原生与 amd64 制品阶段记录为 skipped/waived,并直接使用当前生产脱敏快照进入线上模拟。修复线上报告合并未写入生产 Run ID 导致后续 promote CAS 必然失败的问题。已通过报告回归测试、bash -n、ShellCheck 和 diff 检查。
This commit is contained in:
2026-07-31 23:15:22 +08:00
parent a95184b5b6
commit 0b681275ed
4 changed files with 147 additions and 29 deletions
+10
View File
@@ -132,6 +132,16 @@ scripts/cluster/run-production-acceptance.sh \
--local-report dist/acceptance/local/<本地run-id>/acceptance-report.json --local-report dist/acceptance/local/<本地run-id>/acceptance-report.json
``` ```
只有用户明确要求跳过本地验收时,才允许使用下面的审计豁免。报告会把 `localNative`
`amd64Artifact` 记录为 `skipped`,并保留 `local_acceptance_explicitly_waived` 记录;不得
伪造为通过。线上模拟、真实金丝雀、资源、一致性和发布 CAS 门禁仍全部执行:
```bash
scripts/cluster/run-production-acceptance.sh \
--execute dist/releases/<完整SHA>.json \
--skip-local-acceptance
```
脚本依次执行: 脚本依次执行:
1. 校验工作区、manifest、线上 release SHA、digest、双站点镜像和副本数。 1. 校验工作区、manifest、线上 release SHA、digest、双站点镜像和副本数。
+59 -3
View File
@@ -86,13 +86,66 @@ function validate(report) {
throw new Error('acceptance report stages, gates, or promotion are invalid'); throw new Error('acceptance report stages, gates, or promotion are invalid');
} }
for (const [name, stage] of Object.entries(report.stages)) { for (const [name, stage] of Object.entries(report.stages)) {
if (!['passed', 'failed', 'not-run'].includes(stage?.status)) { if (!['passed', 'failed', 'not-run', 'skipped'].includes(stage?.status)) {
throw new Error(`invalid stage status for ${name}`); throw new Error(`invalid stage status for ${name}`);
} }
} }
assertSecretSafe(report); assertSecretSafe(report);
} }
async function buildOnlineWaiver(values) {
for (const key of ['release-sha', 'api-digest', 'snapshot', 'waiver-reason', 'output']) {
if (!values[key]) throw new Error(`--${key} is required`);
}
const snapshot = await regularJSON(values.snapshot);
if (!shaPattern.test(values['release-sha']) || !digestPattern.test(values['api-digest']) ||
snapshot.schemaVersion !== 'acceptance-snapshot/v1' ||
!shaPattern.test(snapshot.source?.releaseSha ?? '') ||
!hashPattern.test(snapshot.source?.configHash ?? '') ||
!hashPattern.test(snapshot.snapshotSha256 ?? '') ||
!/^[a-z0-9_]{3,80}$/.test(values['waiver-reason'])) {
throw new Error('online waiver identity fields or reason are invalid');
}
const report = {
schemaVersion: 'acceptance-report/v1',
runId: `online-waiver-${values['release-sha'].slice(0, 12)}`,
release: {
sourceSha: values['release-sha'],
apiImageDigest: values['api-digest'],
workerImageDigest: values['api-digest']
},
snapshot: {
sourceReleaseSha: snapshot.source.releaseSha,
configHash: snapshot.source.configHash,
sha256: snapshot.snapshotSha256
},
stages: {
localNative: { status: 'skipped', waiverReason: values['waiver-reason'] },
amd64Artifact: { status: 'skipped', waiverReason: values['waiver-reason'] },
onlineSimulation: { status: 'not-run' },
realCanary: { status: 'not-run' }
},
certifiedProfile: null,
gates: [
{
id: 'local_acceptance_explicitly_waived',
passed: true,
detail: values['waiver-reason']
}
],
promotion: {
ready: false,
requiresManualConfirmation: true,
trafficMode: 'not-applicable'
},
createdAt: new Date().toISOString(),
secretSafe: true
};
validate(report);
await writeNewJSON(values.output, report);
process.stdout.write('acceptance_report_online_waiver=PASS\n');
}
async function loadReports(directory) { async function loadReports(directory) {
const entries = (await readdir(resolve(directory), { withFileTypes: true })) const entries = (await readdir(resolve(directory), { withFileTypes: true }))
.filter((entry) => entry.isFile() && entry.name.endsWith('.json')) .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
@@ -260,12 +313,13 @@ async function mergeProduction(values) {
const local = await regularJSON(values['local-report']); const local = await regularJSON(values['local-report']);
const production = await regularJSON(values['production-summary']); const production = await regularJSON(values['production-summary']);
validate(local); validate(local);
if (local.release.sourceSha !== production.releaseSha || if (!production.runId || local.release.sourceSha !== production.releaseSha ||
local.release.apiImageDigest !== production.apiImageDigest || local.release.apiImageDigest !== production.apiImageDigest ||
local.snapshot.configHash !== production.snapshotConfigHash || local.snapshot.configHash !== production.snapshotConfigHash ||
local.snapshot.sha256 !== production.snapshotSha256) { local.snapshot.sha256 !== production.snapshotSha256) {
throw new Error('production summary does not match local acceptance CAS fields'); throw new Error('production summary does not match local acceptance CAS fields');
} }
local.runId = production.runId;
local.stages.onlineSimulation = { local.stages.onlineSimulation = {
status: production.passed === true ? 'passed' : 'failed', status: production.passed === true ? 'passed' : 'failed',
stableCapacityProfile: production.stableCapacityProfile, stableCapacityProfile: production.stableCapacityProfile,
@@ -299,6 +353,8 @@ if (command === 'validate') {
await buildLocal(values); await buildLocal(values);
} else if (command === 'build-local-partial') { } else if (command === 'build-local-partial') {
await buildLocalPartial(values); await buildLocalPartial(values);
} else if (command === 'build-online-waiver') {
await buildOnlineWaiver(values);
} else if (command === 'merge-production') { } else if (command === 'merge-production') {
await mergeProduction(values); await mergeProduction(values);
} else if (command === 'mark-promoted') { } else if (command === 'mark-promoted') {
@@ -338,5 +394,5 @@ if (command === 'validate') {
await replaceJSONAtomically(input, output, report); await replaceJSONAtomically(input, output, report);
process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`); process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`);
} else { } else {
throw new Error('usage: report.mjs {validate|build-local|build-local-partial|merge-production|mark-promoted|attach-monitor} [options]'); throw new Error('usage: report.mjs {validate|build-local|build-local-partial|build-online-waiver|merge-production|mark-promoted|attach-monitor} [options]');
} }
+59 -26
View File
@@ -12,6 +12,9 @@ Usage:
scripts/cluster/run-production-acceptance.sh \ scripts/cluster/run-production-acceptance.sh \
--execute dist/releases/<SHA>.json \ --execute dist/releases/<SHA>.json \
--local-report dist/acceptance/local/<run-id>/acceptance-report.json --local-report dist/acceptance/local/<run-id>/acceptance-report.json
scripts/cluster/run-production-acceptance.sh \
--execute dist/releases/<SHA>.json \
--skip-local-acceptance
scripts/cluster/run-production-acceptance.sh \ scripts/cluster/run-production-acceptance.sh \
--promote dist/releases/<SHA>.json --run-id <production-run-id> --promote dist/releases/<SHA>.json --run-id <production-run-id>
@@ -22,6 +25,9 @@ access to the selected production candidates. A failed run deliberately keeps
traffic in validation mode and restores only the last stable capacity profile. traffic in validation mode and restores only the last stable capacity profile.
Successful execute runs also remain in validation. Only the separate --promote Successful execute runs also remain in validation. Only the separate --promote
action performs the final CAS switch to live after manual confirmation. action performs the final CAS switch to live after manual confirmation.
The explicit --skip-local-acceptance form records both local stages as skipped
with a user-directed waiver. It never records them as passed and preserves all
online simulation, real canary, resource, consistency, and release CAS gates.
Required private environment values: Required private environment values:
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
@@ -44,21 +50,28 @@ if [[ ${1:-} == --promote ]]; then
exec "$script_dir/promote-production-acceptance.sh" "$@" exec "$script_dir/promote-production-acceptance.sh" "$@"
fi fi
[[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]] || { skip_local_acceptance=false
local_acceptance_report=
if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
release_manifest=$2
local_acceptance_report=$4
elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then
release_manifest=$2
skip_local_acceptance=true
else
usage >&2 usage >&2
exit 64 exit 64
} fi
release_manifest=$2
local_acceptance_report=$4
[[ -f $release_manifest && ! -L $release_manifest ]] || { [[ -f $release_manifest && ! -L $release_manifest ]] || {
echo 'release manifest must be a regular file' >&2 echo 'release manifest must be a regular file' >&2
exit 1 exit 1
} }
[[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || { if [[ $skip_local_acceptance != true ]]; then
echo 'local acceptance report must be a regular file' >&2 [[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || {
exit 1 echo 'local acceptance report must be a regular file' >&2
} exit 1
}
fi
load_cluster_env load_cluster_env
require_commands curl git go jq node openssl sed require_commands curl git go jq node openssl sed
@@ -136,17 +149,19 @@ api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_mani
} }
api_digest=${api_image##*@} api_digest=${api_image##*@}
worker_digest=$api_digest worker_digest=$api_digest
node "$cluster_root/scripts/acceptance/report.mjs" validate \ if [[ $skip_local_acceptance != true ]]; then
--input "$local_acceptance_report" >/dev/null node "$cluster_root/scripts/acceptance/report.mjs" validate \
[[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" && --input "$local_acceptance_report" >/dev/null
$(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" && [[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" &&
$(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed && $(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" &&
$(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed && $(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed &&
$(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run && $(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed &&
$(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || { $(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run &&
echo 'local acceptance report does not match this release or has invalid stage state' >&2 $(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || {
exit 1 echo 'local acceptance report does not match this release or has invalid stage state' >&2
} exit 1
}
fi
[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" ]] || { [[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" ]] || {
echo 'working copy HEAD must equal the release SHA' >&2 echo 'working copy HEAD must equal the release SHA' >&2
exit 1 exit 1
@@ -185,8 +200,12 @@ AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
current_production_snapshot=$temporary_root/current-production-snapshot.json current_production_snapshot=$temporary_root/current-production-snapshot.json
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report") local_snapshot_config_hash=
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report") local_snapshot_sha256=
if [[ $skip_local_acceptance != true ]]; then
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
fi
active_load_pid= active_load_pid=
active_pressure_pid= active_pressure_pid=
capacity_profiles_payload= capacity_profiles_payload=
@@ -232,10 +251,22 @@ AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
"$acceptance_snapshot_binary" export \ "$acceptance_snapshot_binary" export \
--release-sha "$release_sha" \ --release-sha "$release_sha" \
--output "$current_production_snapshot" >/dev/null --output "$current_production_snapshot" >/dev/null
[[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || { if [[ $skip_local_acceptance == true ]]; then
echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2 local_acceptance_report=$temporary_root/online-waiver-base.json
exit 1 node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \
} --release-sha "$release_sha" \
--api-digest "$api_digest" \
--snapshot "$current_production_snapshot" \
--waiver-reason user_directed_online_only_acceptance \
--output "$local_acceptance_report" >/dev/null
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
else
[[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || {
echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2
exit 1
}
fi
remote_kubectl() { remote_kubectl() {
local command_text local command_text
@@ -1073,6 +1104,7 @@ mark_run_failed() {
} }
}') }')
jq -n \ jq -n \
--arg runId "$run_id" \
--arg releaseSha "$release_sha" \ --arg releaseSha "$release_sha" \
--arg apiImageDigest "$api_digest" \ --arg apiImageDigest "$api_digest" \
--arg snapshotConfigHash "$local_snapshot_config_hash" \ --arg snapshotConfigHash "$local_snapshot_config_hash" \
@@ -1082,6 +1114,7 @@ mark_run_failed() {
--arg reason "$reason" \ --arg reason "$reason" \
--argjson artifacts "$artifact_names" \ --argjson artifacts "$artifact_names" \
'{ '{
runId:$runId,
releaseSha:$releaseSha, releaseSha:$releaseSha,
apiImageDigest:$apiImageDigest, apiImageDigest:$apiImageDigest,
snapshotConfigHash:$snapshotConfigHash, snapshotConfigHash:$snapshotConfigHash,
+19
View File
@@ -22,6 +22,7 @@ const output = join(temporary, 'acceptance-report.json');
const partialOutput = join(temporary, 'acceptance-report.partial.json'); const partialOutput = join(temporary, 'acceptance-report.partial.json');
const productionPartial = join(temporary, 'production-summary.partial.json'); const productionPartial = join(temporary, 'production-summary.partial.json');
const mergedPartialOutput = join(temporary, 'acceptance-report.production-partial.json'); const mergedPartialOutput = join(temporary, 'acceptance-report.production-partial.json');
const waivedOutput = join(temporary, 'acceptance-report.waived.json');
writeJSON(runtime, { writeJSON(runtime, {
schemaVersion: 'acceptance-runtime/v1', schemaVersion: 'acceptance-runtime/v1',
runId: runID, runId: runID,
@@ -108,6 +109,7 @@ assert.equal(readFileSync(partialOutput, 'utf8').includes('must-not-be-copied'),
execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', partialOutput]); execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', partialOutput]);
writeJSON(productionPartial, { writeJSON(productionPartial, {
runId: runID,
releaseSha, releaseSha,
apiImageDigest: digest, apiImageDigest: digest,
snapshotConfigHash: configHash, snapshotConfigHash: configHash,
@@ -132,8 +134,25 @@ execFileSync('node', [
]); ]);
const mergedPartial = JSON.parse(readFileSync(mergedPartialOutput, 'utf8')); const mergedPartial = JSON.parse(readFileSync(mergedPartialOutput, 'utf8'));
assert.equal(mergedPartial.stages.onlineSimulation.status, 'failed'); assert.equal(mergedPartial.stages.onlineSimulation.status, 'failed');
assert.equal(mergedPartial.runId, runID);
assert.equal(mergedPartial.stages.realCanary.status, 'failed'); assert.equal(mergedPartial.stages.realCanary.status, 'failed');
assert.equal(mergedPartial.promotion.ready, false); assert.equal(mergedPartial.promotion.ready, false);
assert.equal(mergedPartial.promotion.trafficMode, 'validation'); assert.equal(mergedPartial.promotion.trafficMode, 'validation');
assert.equal(mergedPartial.gates.at(-1).id, 'workflow_interrupted'); assert.equal(mergedPartial.gates.at(-1).id, 'workflow_interrupted');
execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', mergedPartialOutput]); execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', mergedPartialOutput]);
execFileSync('node', [
join(root, 'scripts/acceptance/report.mjs'),
'build-online-waiver',
'--release-sha', releaseSha,
'--api-digest', digest,
'--snapshot', snapshot,
'--waiver-reason', 'user_directed_online_only_acceptance',
'--output', waivedOutput
]);
const waived = JSON.parse(readFileSync(waivedOutput, 'utf8'));
assert.equal(waived.stages.localNative.status, 'skipped');
assert.equal(waived.stages.amd64Artifact.status, 'skipped');
assert.equal(waived.gates[0].id, 'local_acceptance_explicitly_waived');
assert.equal(waived.promotion.ready, false);
execFileSync('node', [join(root, 'scripts/acceptance/report.mjs'), 'validate', '--input', waivedOutput]);