fix(acceptance): 支持显式跳过本地验收
在用户明确授权时,将本地原生与 amd64 制品阶段记录为 skipped/waived,并直接使用当前生产脱敏快照进入线上模拟。修复线上报告合并未写入生产 Run ID 导致后续 promote CAS 必然失败的问题。已通过报告回归测试、bash -n、ShellCheck 和 diff 检查。
This commit is contained in:
@@ -86,13 +86,66 @@ function validate(report) {
|
||||
throw new Error('acceptance report stages, gates, or promotion are invalid');
|
||||
}
|
||||
for (const [name, stage] of Object.entries(report.stages)) {
|
||||
if (!['passed', 'failed', 'not-run'].includes(stage?.status)) {
|
||||
if (!['passed', 'failed', 'not-run', 'skipped'].includes(stage?.status)) {
|
||||
throw new Error(`invalid stage status for ${name}`);
|
||||
}
|
||||
}
|
||||
assertSecretSafe(report);
|
||||
}
|
||||
|
||||
async function buildOnlineWaiver(values) {
|
||||
for (const key of ['release-sha', 'api-digest', 'snapshot', 'waiver-reason', 'output']) {
|
||||
if (!values[key]) throw new Error(`--${key} is required`);
|
||||
}
|
||||
const snapshot = await regularJSON(values.snapshot);
|
||||
if (!shaPattern.test(values['release-sha']) || !digestPattern.test(values['api-digest']) ||
|
||||
snapshot.schemaVersion !== 'acceptance-snapshot/v1' ||
|
||||
!shaPattern.test(snapshot.source?.releaseSha ?? '') ||
|
||||
!hashPattern.test(snapshot.source?.configHash ?? '') ||
|
||||
!hashPattern.test(snapshot.snapshotSha256 ?? '') ||
|
||||
!/^[a-z0-9_]{3,80}$/.test(values['waiver-reason'])) {
|
||||
throw new Error('online waiver identity fields or reason are invalid');
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 'acceptance-report/v1',
|
||||
runId: `online-waiver-${values['release-sha'].slice(0, 12)}`,
|
||||
release: {
|
||||
sourceSha: values['release-sha'],
|
||||
apiImageDigest: values['api-digest'],
|
||||
workerImageDigest: values['api-digest']
|
||||
},
|
||||
snapshot: {
|
||||
sourceReleaseSha: snapshot.source.releaseSha,
|
||||
configHash: snapshot.source.configHash,
|
||||
sha256: snapshot.snapshotSha256
|
||||
},
|
||||
stages: {
|
||||
localNative: { status: 'skipped', waiverReason: values['waiver-reason'] },
|
||||
amd64Artifact: { status: 'skipped', waiverReason: values['waiver-reason'] },
|
||||
onlineSimulation: { status: 'not-run' },
|
||||
realCanary: { status: 'not-run' }
|
||||
},
|
||||
certifiedProfile: null,
|
||||
gates: [
|
||||
{
|
||||
id: 'local_acceptance_explicitly_waived',
|
||||
passed: true,
|
||||
detail: values['waiver-reason']
|
||||
}
|
||||
],
|
||||
promotion: {
|
||||
ready: false,
|
||||
requiresManualConfirmation: true,
|
||||
trafficMode: 'not-applicable'
|
||||
},
|
||||
createdAt: new Date().toISOString(),
|
||||
secretSafe: true
|
||||
};
|
||||
validate(report);
|
||||
await writeNewJSON(values.output, report);
|
||||
process.stdout.write('acceptance_report_online_waiver=PASS\n');
|
||||
}
|
||||
|
||||
async function loadReports(directory) {
|
||||
const entries = (await readdir(resolve(directory), { withFileTypes: true }))
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
|
||||
@@ -260,12 +313,13 @@ async function mergeProduction(values) {
|
||||
const local = await regularJSON(values['local-report']);
|
||||
const production = await regularJSON(values['production-summary']);
|
||||
validate(local);
|
||||
if (local.release.sourceSha !== production.releaseSha ||
|
||||
if (!production.runId || local.release.sourceSha !== production.releaseSha ||
|
||||
local.release.apiImageDigest !== production.apiImageDigest ||
|
||||
local.snapshot.configHash !== production.snapshotConfigHash ||
|
||||
local.snapshot.sha256 !== production.snapshotSha256) {
|
||||
throw new Error('production summary does not match local acceptance CAS fields');
|
||||
}
|
||||
local.runId = production.runId;
|
||||
local.stages.onlineSimulation = {
|
||||
status: production.passed === true ? 'passed' : 'failed',
|
||||
stableCapacityProfile: production.stableCapacityProfile,
|
||||
@@ -299,6 +353,8 @@ if (command === 'validate') {
|
||||
await buildLocal(values);
|
||||
} else if (command === 'build-local-partial') {
|
||||
await buildLocalPartial(values);
|
||||
} else if (command === 'build-online-waiver') {
|
||||
await buildOnlineWaiver(values);
|
||||
} else if (command === 'merge-production') {
|
||||
await mergeProduction(values);
|
||||
} else if (command === 'mark-promoted') {
|
||||
@@ -338,5 +394,5 @@ if (command === 'validate') {
|
||||
await replaceJSONAtomically(input, output, report);
|
||||
process.stdout.write(`acceptance_report_monitor=PASS monitor_passed=${monitor.passed === true}\n`);
|
||||
} else {
|
||||
throw new Error('usage: report.mjs {validate|build-local|build-local-partial|merge-production|mark-promoted|attach-monitor} [options]');
|
||||
throw new Error('usage: report.mjs {validate|build-local|build-local-partial|build-online-waiver|merge-production|mark-promoted|attach-monitor} [options]');
|
||||
}
|
||||
|
||||
@@ -12,6 +12,9 @@ Usage:
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--execute dist/releases/<SHA>.json \
|
||||
--local-report dist/acceptance/local/<run-id>/acceptance-report.json
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--execute dist/releases/<SHA>.json \
|
||||
--skip-local-acceptance
|
||||
scripts/cluster/run-production-acceptance.sh \
|
||||
--promote dist/releases/<SHA>.json --run-id <production-run-id>
|
||||
|
||||
@@ -22,6 +25,9 @@ access to the selected production candidates. A failed run deliberately keeps
|
||||
traffic in validation mode and restores only the last stable capacity profile.
|
||||
Successful execute runs also remain in validation. Only the separate --promote
|
||||
action performs the final CAS switch to live after manual confirmation.
|
||||
The explicit --skip-local-acceptance form records both local stages as skipped
|
||||
with a user-directed waiver. It never records them as passed and preserves all
|
||||
online simulation, real canary, resource, consistency, and release CAS gates.
|
||||
|
||||
Required private environment values:
|
||||
AI_GATEWAY_ACCEPTANCE_ADMIN_TOKEN
|
||||
@@ -44,21 +50,28 @@ if [[ ${1:-} == --promote ]]; then
|
||||
exec "$script_dir/promote-production-acceptance.sh" "$@"
|
||||
fi
|
||||
|
||||
[[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]] || {
|
||||
skip_local_acceptance=false
|
||||
local_acceptance_report=
|
||||
if [[ ${1:-} == --execute && ${3:-} == --local-report && $# -eq 4 ]]; then
|
||||
release_manifest=$2
|
||||
local_acceptance_report=$4
|
||||
elif [[ ${1:-} == --execute && ${3:-} == --skip-local-acceptance && $# -eq 3 ]]; then
|
||||
release_manifest=$2
|
||||
skip_local_acceptance=true
|
||||
else
|
||||
usage >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
release_manifest=$2
|
||||
local_acceptance_report=$4
|
||||
fi
|
||||
[[ -f $release_manifest && ! -L $release_manifest ]] || {
|
||||
echo 'release manifest must be a regular file' >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || {
|
||||
echo 'local acceptance report must be a regular file' >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ $skip_local_acceptance != true ]]; then
|
||||
[[ -f $local_acceptance_report && ! -L $local_acceptance_report ]] || {
|
||||
echo 'local acceptance report must be a regular file' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
load_cluster_env
|
||||
require_commands curl git go jq node openssl sed
|
||||
@@ -136,17 +149,19 @@ api_image=$(node "$cluster_root/scripts/release-manifest.mjs" get "$release_mani
|
||||
}
|
||||
api_digest=${api_image##*@}
|
||||
worker_digest=$api_digest
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" validate \
|
||||
--input "$local_acceptance_report" >/dev/null
|
||||
[[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" &&
|
||||
$(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" &&
|
||||
$(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed &&
|
||||
$(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed &&
|
||||
$(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run &&
|
||||
$(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || {
|
||||
echo 'local acceptance report does not match this release or has invalid stage state' >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ $skip_local_acceptance != true ]]; then
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" validate \
|
||||
--input "$local_acceptance_report" >/dev/null
|
||||
[[ $(jq -r '.release.sourceSha' "$local_acceptance_report") == "$release_sha" &&
|
||||
$(jq -r '.release.apiImageDigest' "$local_acceptance_report") == "$api_digest" &&
|
||||
$(jq -r '.stages.localNative.status' "$local_acceptance_report") == passed &&
|
||||
$(jq -r '.stages.amd64Artifact.status' "$local_acceptance_report") == passed &&
|
||||
$(jq -r '.stages.onlineSimulation.status' "$local_acceptance_report") == not-run &&
|
||||
$(jq -r '.promotion.ready' "$local_acceptance_report") == false ]] || {
|
||||
echo 'local acceptance report does not match this release or has invalid stage state' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
[[ $(git -C "$cluster_root" rev-parse HEAD) == "$release_sha" ]] || {
|
||||
echo 'working copy HEAD must equal the release SHA' >&2
|
||||
exit 1
|
||||
@@ -185,8 +200,12 @@ AI_GATEWAY_ACCEPTANCE_API_KEYS=$AI_GATEWAY_ACCEPTANCE_API_KEY
|
||||
acceptance_load_binary=$temporary_root/easyai-ai-gateway-acceptance-load
|
||||
acceptance_snapshot_binary=$temporary_root/easyai-ai-gateway-acceptance-snapshot
|
||||
current_production_snapshot=$temporary_root/current-production-snapshot.json
|
||||
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
|
||||
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
|
||||
local_snapshot_config_hash=
|
||||
local_snapshot_sha256=
|
||||
if [[ $skip_local_acceptance != true ]]; then
|
||||
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
|
||||
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
|
||||
fi
|
||||
active_load_pid=
|
||||
active_pressure_pid=
|
||||
capacity_profiles_payload=
|
||||
@@ -232,10 +251,22 @@ AI_GATEWAY_DATABASE_URL="$AI_GATEWAY_ACCEPTANCE_SNAPSHOT_DATABASE_URL" \
|
||||
"$acceptance_snapshot_binary" export \
|
||||
--release-sha "$release_sha" \
|
||||
--output "$current_production_snapshot" >/dev/null
|
||||
[[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || {
|
||||
echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ $skip_local_acceptance == true ]]; then
|
||||
local_acceptance_report=$temporary_root/online-waiver-base.json
|
||||
node "$cluster_root/scripts/acceptance/report.mjs" build-online-waiver \
|
||||
--release-sha "$release_sha" \
|
||||
--api-digest "$api_digest" \
|
||||
--snapshot "$current_production_snapshot" \
|
||||
--waiver-reason user_directed_online_only_acceptance \
|
||||
--output "$local_acceptance_report" >/dev/null
|
||||
local_snapshot_config_hash=$(jq -r '.snapshot.configHash' "$local_acceptance_report")
|
||||
local_snapshot_sha256=$(jq -r '.snapshot.sha256' "$local_acceptance_report")
|
||||
else
|
||||
[[ $(jq -r '.source.configHash' "$current_production_snapshot") == "$local_snapshot_config_hash" ]] || {
|
||||
echo 'production candidate configuration changed after the local snapshot; rerun local acceptance' >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
remote_kubectl() {
|
||||
local command_text
|
||||
@@ -1073,6 +1104,7 @@ mark_run_failed() {
|
||||
}
|
||||
}')
|
||||
jq -n \
|
||||
--arg runId "$run_id" \
|
||||
--arg releaseSha "$release_sha" \
|
||||
--arg apiImageDigest "$api_digest" \
|
||||
--arg snapshotConfigHash "$local_snapshot_config_hash" \
|
||||
@@ -1082,6 +1114,7 @@ mark_run_failed() {
|
||||
--arg reason "$reason" \
|
||||
--argjson artifacts "$artifact_names" \
|
||||
'{
|
||||
runId:$runId,
|
||||
releaseSha:$releaseSha,
|
||||
apiImageDigest:$apiImageDigest,
|
||||
snapshotConfigHash:$snapshotConfigHash,
|
||||
|
||||
Reference in New Issue
Block a user