diff --git a/README.md b/README.md index ddc274a..68632ec 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,12 @@ UPDATE_SERVICES='easyai-server easyai-web model-conversion-worker' ./update.sh `MODEL_CONVERSION_WORKER_VERSION` 可以固定已发布的版本；默认 `latest`。混元付费通道默认关闭，自建转换失败不自动切换到付费服务。已有 updater 的页面更新仍按稳定清单管理三个核心服务；转换 Worker 的首次安装和配置升级使用上述 `update` 脚本完成。 +测试候选版本时使用 `UPDATE_RELEASE_VERSION=3.23.0.20261003`，由更新脚本固定核心服务和转换 Worker 的版本，无需改动远端 `.env` 或移动 `latest` 标签。版本号不合法时在修改配置前终止。 + +Linux 旧部署如果因本地 Compose 修改无法拉取仓库，可在检查差异后设置 `UPDATE_ADOPT_LEGACY_CONFIG=1`。脚本需要 Python 3 和 PyYAML；先校验合并配置，再将原始配置和冲突文件备份到 `data/deployment-backups/`，本地差异保存在 `data/deployment-config/legacy-compose.json`，仓库继续管理新增服务和配置。其他业务代码修改、未支持的文件冲突或无法表示的删除会阻止迁移，不启动容器。活动 Nginx 配置不会被改写。 + +没有自更新能力的历史脚本可先通过 Git 获取新版 `update.sh` 和 `scripts/` 到临时目录，再以 `UPDATE_PROJECT_DIR=/实际/easyai/目录` 执行新版脚本；配置迁移、仓库更新和容器操作仍由 `update.sh` 完成。已有 managed updater 应使用仓库分配的独立内网地址，避免在主后端暂时停止时占用其固定地址。 + - **查看帮助**：`./update.sh -h` 或 `./update.sh --help` ### 更新说明 diff --git a/scripts/prepare-update-repository.py b/scripts/prepare-update-repository.py new file mode 100644 index 0000000..0ad1009 --- /dev/null +++ b/scripts/prepare-update-repository.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Adopt reviewed legacy deployment edits into a local Compose override. + +Only deployment configuration is accepted. Business code and arbitrary untracked +files remain untouched. Backups contain private settings and use mode 0700/0600. +""" +import copy +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time + +try: + import yaml +except ImportError: + sys.exit('❌ 迁移旧部署配置需要 Python PyYAML；尚未修改配置或容器') + +root = Path(sys.argv[1]).resolve() +os.chdir(root) + + +def git(*args): + return subprocess.check_output(['git', *args], text=True) + + +upstream = git('rev-parse', '--abbrev-ref', '@{upstream}').strip() +subprocess.run(['git', 'fetch', '--quiet', upstream.split('/')[0]], check=True) +subprocess.run(['git', 'merge-base', '--is-ancestor', 'HEAD', upstream], check=True) +changed = set(git('diff', 'HEAD', '--name-only').splitlines()) +if changed - {'docker-compose.yml'}: + sys.exit('❌ 存在部署配置以外的本地代码修改；尚未修改配置或容器') +tracked = set(git('ls-tree', '-r', '--name-only', upstream).splitlines()) +untracked = set(git('ls-files', '--others', '--exclude-standard').splitlines()) +collisions = tracked & untracked +allowed = {'scripts/init-security-env.sh'} +if any(p not in allowed and not (Path(p).parent == Path('.') and p.endswith('.conf')) for p in collisions): + sys.exit('❌ 存在新版仓库会覆盖的其他本地文件；尚未修改配置或容器') +if not changed and not collisions: + sys.exit(0) +for name in changed | collisions: + if not (root / name).is_file() or (root / name).is_symlink(): + sys.exit('❌ 配置文件类型不受支持；尚未修改配置或容器') + + +MISSING = object() + + +def clone(value): + return MISSING if value is MISSING else copy.deepcopy(value) + + +def volume_target(value): + # Ignore colons inside ${VARIABLE:-default} when identifying the mount target. + depth = 0 + separators = [] + for i, char in enumerate(value): + if char == '{': depth += 1 + elif char == '}': depth -= 1 + elif char == ':' and depth == 0: separators.append(i) + if not separators: return value + end = separators[1] if len(separators) > 1 else len(value) + return value[separators[0] + 1:end] + + +def canonical(value, key=''): + if isinstance(value, dict): + return {k: canonical(v, k) for k, v in value.items()} + if isinstance(value, list): + if key in ('environment', 'labels'): + return dict((v.partition('=')[0], v.partition('=')[2] if '=' in v else None) for v in value) + if key == 'networks': return {v: {} for v in value} + if key == 'volumes' and all(isinstance(v, str) for v in value): + return {volume_target(v): v for v in value} + return value + + +preserved = [] + + +def merge(base, local, remote, path=()): + if local == base: return clone(remote) + if local == remote: return clone(remote) + if isinstance(local, dict) and isinstance(remote, dict) and (isinstance(base, dict) or base is MISSING): + old = base if isinstance(base, dict) else {} + result = {} + for key in sorted(set(old) | set(local) | set(remote)): + value = merge(old.get(key, MISSING), local.get(key, MISSING), remote.get(key, MISSING), path + (key,)) + if value is not MISSING: result[key] = value + return result + if local is MISSING and base is MISSING: return clone(remote) + if local is MISSING: return MISSING + if remote is MISSING and base is not MISSING: + sys.exit('❌ 新版删除了仍被本地修改的字段，无法安全合并；尚未修改配置或容器') + preserved.append('.'.join(path)) + return clone(local) + + +def delta(remote, merged): + if remote == merged: return MISSING + if isinstance(remote, dict) and isinstance(merged, dict): + result = {} + for key in merged: + value = delta(remote.get(key, MISSING), merged[key]) + if value is not MISSING: result[key] = value + # Removal cannot be safely represented by an ordinary Compose override. + if set(remote) - set(merged): + sys.exit('❌ 本地配置删除了新版字段，无法安全合并；尚未修改配置或容器') + return result + return copy.deepcopy(merged) + + +def serialize(value, key=''): + if isinstance(value, dict): + if key == 'volumes': return list(value.values()) + return {k: serialize(v, k) for k, v in value.items()} + return value + + +base = canonical(yaml.safe_load(git('show', 'HEAD:docker-compose.yml'))) +local = canonical(yaml.safe_load((root / 'docker-compose.yml').read_text())) +remote_text = git('show', upstream + ':docker-compose.yml') +remote = canonical(yaml.safe_load(remote_text)) +merged = merge(base, local, remote) +override = delta(remote, merged) +with tempfile.NamedTemporaryFile(mode='w', dir=root, prefix='.update-compose-', suffix='.json', delete=False) as candidate: + json.dump(serialize(merged), candidate) + candidate_path = Path(candidate.name) +try: + candidate_path.chmod(0o600) + compose = ['docker', 'compose'] if shutil.which('docker') else ['docker-compose'] + validation = subprocess.run(compose + ['--env-file', '.env', '-f', str(candidate_path), 'config', '--quiet'], capture_output=True, text=True) + if validation.returncode: + sys.exit('❌ 合并后的部署配置未通过校验；尚未修改配置或容器') +finally: + candidate_path.unlink() +backup_root = root / 'data/deployment-backups' +backup_root.mkdir(parents=True, exist_ok=True, mode=0o700) +backup = Path(tempfile.mkdtemp(prefix='legacy-update-' + time.strftime('%Y%m%d-%H%M%S') + '-', dir=backup_root)) +backup.chmod(0o700) +for name in changed | collisions | {'.env'}: + destination = backup / name + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + shutil.copy2(root / name, destination) + destination.chmod(0o600) +config_dir = root / 'data/deployment-config' +config_dir.mkdir(parents=True, exist_ok=True, mode=0o700) +override_path = config_dir / 'legacy-compose.json' +if override is not MISSING and override: + if override_path.exists(): + sys.exit('❌ 已存在旧配置迁移结果；请检查备份后再重试，尚未修改仓库') + override_path.write_text(json.dumps(serialize(override), indent=2) + '\n') + override_path.chmod(0o600) + env_path = root / '.env' + lines = env_path.read_text().splitlines() + existing = next((line.partition('=')[2].strip().strip('"').strip("'") for line in lines if line.startswith('COMPOSE_FILE=')), '') + files = existing.split(os.pathsep) if existing else ['docker-compose.yml'] + relative = str(override_path.relative_to(root)) + if relative not in files: files.append(relative) + lines = [line for line in lines if not line.startswith('COMPOSE_FILE=')] + lines.append('COMPOSE_FILE=' + os.pathsep.join(files)) + env_path.write_text('\n'.join(lines) + '\n') + env_path.chmod(0o600) +for name in collisions: + (root / name).unlink() +if changed: + subprocess.run(['git', 'restore', '--source=HEAD', '--staged', '--worktree', '--', 'docker-compose.yml'], check=True) +(backup / 'migration.json').write_text(json.dumps({'from': git('rev-parse', 'HEAD').strip(), 'to': git('rev-parse', upstream).strip(), 'preservedFields': preserved, 'files': sorted(changed | collisions)}, indent=2)) +(backup / 'migration.json').chmod(0o600) +print('✅ 已备份旧配置并迁移本地差异；新版部署字段由仓库继续管理') +print(' 备份目录: ' + str(backup)) diff --git a/scripts/test-legacy-deployment-update.py b/scripts/test-legacy-deployment-update.py new file mode 100644 index 0000000..2185225 --- /dev/null +++ b/scripts/test-legacy-deployment-update.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Real Git/config adoption regression; container mutation is not used here.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import time + +source = Path(__file__).resolve().parent.parent +started = time.monotonic() + + +def run(args, root, env=None): + return subprocess.run(args, cwd=root, env=env, capture_output=True, text=True, timeout=30) + + +with tempfile.TemporaryDirectory(prefix='easyai-legacy-update-') as directory: + base = Path(directory) + root = base / 'easyai' + root.mkdir() + (root / 'docker-compose.yml').write_text('services:\n easyai-server:\n image: server:old\n environment:\n - EXISTING=keep\n watchtower:\n image: watcher:old\n') + (root / '.gitignore').write_text('.env\ndata/\n') + (root / '.env').write_text('VERSION=old\nCONFIG_JWT_SECRET=keep-this-private-key\n') + for args in [['git','init','-q'], ['git','config','user.name','Update test'], ['git','config','user.email','test@example.invalid'], ['git','add','.'], ['git','commit','-qm','test: 旧部署基准']]: + assert run(args, root).returncode == 0 + remote = base / 'remote.git' + seed = base / 'seed' + assert run(['git','clone','-q','--bare',str(root),str(remote)], base).returncode == 0 + assert run(['git','remote','add','origin',str(remote)], root).returncode == 0 + assert run(['git','fetch','-q'],root).returncode == 0 + branch = run(['git','branch','--show-current'], root).stdout.strip() + assert run(['git','branch','--set-upstream-to','origin/'+branch],root).returncode == 0 + assert run(['git','clone','-q',str(remote),str(seed)],base).returncode == 0 + (seed / 'docker-compose.yml').write_text('''services: + easyai-server: + image: server:new + environment: + - EXISTING=keep + - MODEL_CONVERSION_WORKER_TOKEN=${MODEL_CONVERSION_WORKER_TOKEN} + watchtower: + image: watcher:new + easyai-updater: + image: updater:new + environment: + - UPDATE_MANIFEST_FILE=/etc/manifest.json + volumes: + - ./data/update-state:/state + - ./system-update/release-manifest.json:/etc/manifest.json:ro + networks: + easyai: + ipv4_address: 172.21.0.31 + model-conversion-worker: + image: worker:new +networks: + easyai: {} +''') + (seed / 'scripts').mkdir() + (seed / 'scripts/init-security-env.sh').write_text('# modern security initializer\n') + (seed / 'demo.example.conf').write_text('# upstream example\n') + for args in [['git','config','user.name','Update test'],['git','config','user.email','test@example.invalid'],['git','add','.'],['git','commit','-qm','test: 新增转换服务与独立更新地址'],['git','push','-q']]: + assert run(args,seed).returncode == 0 + with (root / 'docker-compose.yml').open('a') as stream: + stream.write(''' easyai-updater: + image: updater:new + environment: + - UPDATE_MANIFEST_FILE=/state/custom.json + volumes: + - ./data/update-state:/state + networks: + - easyai +''') + (root / 'scripts').mkdir() + (root / 'scripts/init-security-env.sh').write_text('# local legacy initializer\n') + (root / 'demo.example.conf').write_text('# custom nginx source; active copy elsewhere\n') + original = (root / 'docker-compose.yml').read_bytes() + bins = base / 'bin' + bins.mkdir() + docker = bins / 'docker' + docker.write_text('#!/bin/sh\nexit 0\n') + docker.chmod(0o755) + env = {**os.environ, 'PATH': str(bins)+os.pathsep+os.environ['PATH']} + legacy = run(['git','pull','--ff-only'],root) + assert legacy.returncode != 0 and 'would be overwritten' in legacy.stderr + assert original == (root / 'docker-compose.yml').read_bytes() + (root / 'business.js').write_text('local business code\n') + assert run(['git','add','business.js'],root).returncode == 0 + rejected = run(['python3',str(source/'scripts/prepare-update-repository.py'),str(root)],root,env) + assert rejected.returncode != 0 + assert original == (root / 'docker-compose.yml').read_bytes() + assert run(['git','reset','-q','HEAD','--','business.js'],root).returncode == 0 + accepted = run(['python3',str(source/'scripts/prepare-update-repository.py'),str(root)],root,env) + assert accepted.returncode == 0, accepted.stdout+accepted.stderr + assert run(['git','pull','--ff-only'],root).returncode == 0 + assert run(['git','diff','--name-only'],root).stdout == '' + override=json.loads((root/'data/deployment-config/legacy-compose.json').read_text()) + assert override == {'services': {'easyai-updater': {'environment': {'UPDATE_MANIFEST_FILE': '/state/custom.json'}}}}, override + assert 'COMPOSE_FILE=docker-compose.yml:data/deployment-config/legacy-compose.json' in (root/'.env').read_text() + assert 'CONFIG_JWT_SECRET=keep-this-private-key' in (root/'.env').read_text() + backups=list((root/'data/deployment-backups').glob('legacy-update-*')) + assert len(backups)==1 + assert (backups[0]/'docker-compose.yml').read_bytes()==original + assert (backups[0]/'demo.example.conf').read_text().startswith('# custom nginx') + assert (backups[0]/'.env').stat().st_mode & 0o077 == 0 + assert (backups[0]).stat().st_mode & 0o077 == 0 + resolved=run(['docker','compose','--env-file','.env','config','--format','json'],root) + assert resolved.returncode==0,resolved.stderr + config=json.loads(resolved.stdout) + assert config['services']['easyai-updater']['networks']['easyai']['ipv4_address']=='172.21.0.31' + assert config['services']['easyai-updater']['environment']['UPDATE_MANIFEST_FILE']=='/state/custom.json' + assert 'model-conversion-worker' in config['services'] + repeat=run(['python3',str(source/'scripts/prepare-update-repository.py'),str(root)],root,env) + assert repeat.returncode==0 + assert len(list((root/'data/deployment-backups').glob('legacy-update-*')))==1 + print('legacy Git conflict: reproduced; code guard, private backups, local override, new Worker and updater address passed') +print(json.dumps({'passed':True,'durationSeconds':round(time.monotonic()-started,3)})) diff --git a/scripts/test-model-conversion-update.py b/scripts/test-model-conversion-update.py index 8c40495..b7b417a 100644 --- a/scripts/test-model-conversion-update.py +++ b/scripts/test-model-conversion-update.py @@ -99,6 +99,16 @@ with tempfile.TemporaryDirectory(prefix='easyai-conversion-update-') as temporar assert result.returncode == 0, result.stdout + result.stderr assert original == (root / '.env').read_bytes(), 'repeat changes deployment settings' log.write_text('') + original = (root / '.env').read_bytes() + result = run(command, root, {**env, 'UPDATE_RELEASE_VERSION': 'latest;unsafe'}) + assert result.returncode != 0 + assert original == (root / '.env').read_bytes() + assert not any(c[:2] in [['compose', 'pull'], ['compose', 'up']] for c in calls(log)) + result = run(command, root, {**env, 'UPDATE_RELEASE_VERSION': '3.23.0.20261003'}) + assert result.returncode == 0, result.stdout + result.stderr + values = dict(line.split('=', 1) for line in (root / '.env').read_text().splitlines() if '=' in line and not line.startswith('#')) + assert values['VERSION'] == values['MODEL_CONVERSION_WORKER_VERSION'] == '3.23.0.20261003' + log.write_text('') result = run(command, root, {**env, 'UPDATE_SERVICES': 'easyai-server missing-service'}) assert result.returncode != 0 assert not any(c[:2] in [['compose', 'pull'], ['compose', 'up']] for c in calls(log)) diff --git a/update.ps1 b/update.ps1 index 0b4cc96..ba17edf 100644 --- a/update.ps1 +++ b/update.ps1 @@ -58,6 +58,8 @@ if ($args -contains "-h" -or $args -contains "--help") { Write-Host " false 等同 UPDATE_MODE=full" Write-Host " UPDATE_SERVICES='easyai-server easyai-web model-conversion-worker'" Write-Host " 仅拉取并重建指定服务，不重建其依赖；留空保持整组更新。" + Write-Host " UPDATE_RELEASE_VERSION=固定版本" + Write-Host " 通过脚本固定核心服务和转换 Worker 的镜像版本，不提升 latest。" Write-Host " UPDATE_NO_WAIT=1" Write-Host " 结束或失败时不等待按 Enter" Write-Host " UPDATE_HEALTH_TIMEOUT_SECONDS=秒数" @@ -131,6 +133,10 @@ function Resolve-UpdateMode { $skipRepoUpdate = Resolve-UpdateMode +if ($env:UPDATE_RELEASE_VERSION -and $env:UPDATE_RELEASE_VERSION -notmatch '^\d+\.\d+\.\d+\.\d{8}(\.\d+)?$') { + Write-Err "UPDATE_RELEASE_VERSION 格式不正确；尚未修改配置或容器" +} + # ==================== 拉取仓库 ==================== if (-not $skipRepoUpdate) { Write-Host "" @@ -215,6 +221,14 @@ Initialize-ServerHttpBindIp -Path (Join-Path $scriptDir ".env") . (Join-Path $scriptDir "scripts\Initialize-SecurityEnv.ps1") Initialize-SecurityEnv -Path (Join-Path $scriptDir ".env") -Mode "upgrade" +if ($env:UPDATE_RELEASE_VERSION) { + $environmentPath = Join-Path $scriptDir ".env" + $environmentContent = Get-Content -Raw -Encoding UTF8 $environmentPath + $environmentContent = Set-EnvValue $environmentContent "VERSION" $env:UPDATE_RELEASE_VERSION + $environmentContent = Set-EnvValue $environmentContent "MODEL_CONVERSION_WORKER_VERSION" $env:UPDATE_RELEASE_VERSION + [System.IO.File]::WriteAllText($environmentPath, $environmentContent, (New-Object System.Text.UTF8Encoding $false)) + Write-Step "📌 已固定本次发布版本: $($env:UPDATE_RELEASE_VERSION)" +} # 在更新镜像和重建服务前迁移并校验后端公开 API 地址。 . (Join-Path $scriptDir "scripts\Initialize-PublicApiBaseUrl.ps1") diff --git a/update.sh b/update.sh index 813b1bc..912c70a 100755 --- a/update.sh +++ b/update.sh @@ -19,13 +19,18 @@ if [[ "${1:-}" =~ ^(-h|--help)$ ]]; then echo " 等待核心服务恢复可用的最长时间，默认 300 秒" echo " UPDATE_SERVICES='easyai-server easyai-web model-conversion-worker'" echo " 仅拉取并重建指定服务；不重建其依赖。留空保持整组更新。" + echo " UPDATE_RELEASE_VERSION=固定版本" + echo " 通过脚本固定核心服务和转换 Worker 的镜像版本，不提升 latest。" + echo " UPDATE_ADOPT_LEGACY_CONFIG=1" + echo " 备份旧部署配置并将本地差异迁移为独立覆盖文件。" echo "" echo "仓库更新后会自动执行 scripts/update-env.d 中的幂等环境配置迁移。" exit 0 fi # 进入脚本所在目录（项目根目录） -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +SCRIPT_DIR="$(cd "${UPDATE_PROJECT_DIR:-$SOURCE_DIR}" && pwd)" cd "$SCRIPT_DIR" # 命令行内选择更新方式（默认：更新并拉取仓库） @@ -92,6 +97,11 @@ resolve_update_mode() { resolve_update_mode +if [ -n "${UPDATE_RELEASE_VERSION:-}" ] && ! [[ "$UPDATE_RELEASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]{8}(\.[0-9]+)?$ ]]; then + echo "❌ UPDATE_RELEASE_VERSION 格式不正确；尚未修改配置或容器" >&2 + exit 1 +fi + # 拉取整个仓库更新 if [ "$SKIP_REPO_UPDATE" = false ]; then echo "" @@ -107,6 +117,9 @@ if [ "$SKIP_REPO_UPDATE" = false ]; then fi before_revision="$(git rev-parse HEAD)" + if [ "${UPDATE_ADOPT_LEGACY_CONFIG:-0}" = "1" ]; then + python3 "$SOURCE_DIR/scripts/prepare-update-repository.py" "$SCRIPT_DIR" + fi echo "📥 正在执行 git pull --ff-only..." if git pull --ff-only; then echo "✅ 仓库已更新到最新版本" @@ -161,6 +174,12 @@ init_server_http_bind_ip .env . ./scripts/init-security-env.sh init_security_env .env upgrade +if [ -n "${UPDATE_RELEASE_VERSION:-}" ]; then + write_env_value .env VERSION "$UPDATE_RELEASE_VERSION" + write_env_value .env MODEL_CONVERSION_WORKER_VERSION "$UPDATE_RELEASE_VERSION" + echo "📌 已固定本次发布版本: $UPDATE_RELEASE_VERSION" +fi + # 在更新镜像和重建服务前迁移并校验后端公开 API 地址。 # shellcheck source=scripts/init-public-api-base-url.sh . ./scripts/init-public-api-base-url.sh