From 89a4ccd7e7d71fd6f4635f384b08ae157c6586ab Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 8 Oct 2026 09:23:25 +0800 Subject: [PATCH] =?UTF-8?q?feat(deploy):=20=E4=B8=BA=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E7=BB=84=E4=BB=B6=E5=A2=9E=E5=8A=A0=E5=AE=9A=E6=97=B6=E7=BB=B4?= =?UTF-8?q?=E6=8A=A4=E4=B8=8E=E5=8F=8C=E4=BF=9D=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.sample | 8 +- .../workflows/test-deployment-public-url.yml | 14 +++ .github/workflows/test-start-ps1.yml | 11 ++ README.md | 12 +- docker-compose.yml | 15 ++- .../Initialize-InfrastructureUpdateEnv.ps1 | 20 ++++ scripts/Install-InfrastructureUpdateTask.ps1 | 30 +++++ scripts/Test-InfrastructureUpdateTask.ps1 | 31 +++++ scripts/Update-Infrastructure.ps1 | 74 ++++++++++++ scripts/init-infrastructure-update-env.sh | 20 ++++ scripts/install-infrastructure-update-task.sh | 57 +++++++++ scripts/test-deployment-public-url.sh | 5 + scripts/test-infrastructure-auto-update.py | 112 ++++++++++++++++++ scripts/test-model-conversion-update.py | 2 + scripts/test-update-env-migrations.sh | 2 +- .../update-env.d/020-watchtower-progress.env | 2 +- .../023-infrastructure-auto-update.env | 2 + scripts/update-infrastructure.sh | 81 +++++++++++++ start.ps1 | 3 + start.sh | 3 + .../acceptance.md | 65 ++++++++++ .../after.json | 53 +++++++++ .../before.json | 28 +++++ .../cleanup.json | 10 ++ .../quality-checks.json | 7 ++ .../validation.json | 14 +++ .../acceptance.md | 80 +++++++++++++ .../release-manifest.candidate.json | 37 ++++++ update.ps1 | 3 + update.sh | 2 + 30 files changed, 796 insertions(+), 7 deletions(-) create mode 100644 scripts/Initialize-InfrastructureUpdateEnv.ps1 create mode 100644 scripts/Install-InfrastructureUpdateTask.ps1 create mode 100644 scripts/Test-InfrastructureUpdateTask.ps1 create mode 100644 scripts/Update-Infrastructure.ps1 create mode 100755 scripts/init-infrastructure-update-env.sh create mode 100755 scripts/install-infrastructure-update-task.sh create mode 100644 scripts/test-infrastructure-auto-update.py create mode 100644 scripts/update-env.d/023-infrastructure-auto-update.env create mode 100755 scripts/update-infrastructure.sh create mode 100644 system-update/releases/infrastructure-auto-update-20261008/acceptance.md create mode 100644 system-update/releases/infrastructure-auto-update-20261008/after.json create mode 100644 system-update/releases/infrastructure-auto-update-20261008/before.json create mode 100644 system-update/releases/infrastructure-auto-update-20261008/cleanup.json create mode 100644 system-update/releases/infrastructure-auto-update-20261008/quality-checks.json create mode 100644 system-update/releases/infrastructure-auto-update-20261008/validation.json create mode 100644 system-update/releases/online-update-release-source-20261007/acceptance.md create mode 100644 system-update/releases/online-update-release-source-20261007/release-manifest.candidate.json diff --git a/.env.sample b/.env.sample index 678aa05..7403d4f 100644 --- a/.env.sample +++ b/.env.sample @@ -261,6 +261,9 @@ COMPOSE_PROFILES=managed-update WATCHTOWER_CORE_UPDATE_ENABLED=false # 更新器由独立于任务执行进程的 Watchtower 每 24 小时检查一次,避免自替换中断任务。 WATCHTOWER_UPDATE_TARGET=easyai-updater +UPDATE_MANIFEST_SOURCE=remote +# 留空时使用部署仓库中的官方稳定发布清单;离线部署可显式选择 local。 +UPDATE_MANIFEST_URL= SYSTEM_UPDATER_UPDATE_INTERVAL_SECONDS=86400 WATCHTOWER_CLEANUP=true # 核心服务替换后最多等待 5 分钟恢复可用。 @@ -268,7 +271,10 @@ UPDATE_HEALTH_TIMEOUT_MS=300000 # 私有镜像仓库登录信息;执行 docker login 后通常位于此处。 DOCKER_CONFIG_FILE=/root/.docker/config.json SYSTEM_UPDATE_INTERNAL_TOKEN= -WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3 +# 宿主机每六小时分别维护 Watchtower 和 updater;保留 Watchtower 的 updater 更新计划。 +INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true +WATCHTOWER_SCOPE=easyai +WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest # 更新接口及事件接口仅供容器内网访问,由初始化脚本生成独立令牌。 WATCHTOWER_API_KEY= WATCHTOWER_HTTP_API_EVENTS_TOKEN= diff --git a/.github/workflows/test-deployment-public-url.yml b/.github/workflows/test-deployment-public-url.yml index 08f09a5..823c20b 100644 --- a/.github/workflows/test-deployment-public-url.yml +++ b/.github/workflows/test-deployment-public-url.yml @@ -7,6 +7,10 @@ on: - 'update.sh' - 'docker-compose.yml' - 'easyai-proxy.conf.sample' + - 'scripts/*infrastructure*.sh' + - 'scripts/*Infrastructure*.ps1' + - 'scripts/test-infrastructure-auto-update.py' + - 'scripts/update-env.d/*.env' - '.env.sample' - 'scripts/init-public-api-base-url.sh' - 'scripts/test-public-api-base-url.sh' @@ -25,6 +29,10 @@ on: - 'update.sh' - 'docker-compose.yml' - 'easyai-proxy.conf.sample' + - 'scripts/*infrastructure*.sh' + - 'scripts/*Infrastructure*.ps1' + - 'scripts/test-infrastructure-auto-update.py' + - 'scripts/update-env.d/*.env' - '.env.sample' - 'scripts/init-public-api-base-url.sh' - 'scripts/test-public-api-base-url.sh' @@ -62,3 +70,9 @@ jobs: - name: Test deployment dry-run matrix run: bash scripts/test-deployment-public-url.sh + + - name: Test automatic Watchtower maintenance + run: | + python3 scripts/test-infrastructure-auto-update.py + pwsh -NoProfile -File scripts/Test-InfrastructureUpdateTask.ps1 + bash scripts/test-update-env-migrations.sh diff --git a/.github/workflows/test-start-ps1.yml b/.github/workflows/test-start-ps1.yml index 4a76904..d452f2a 100644 --- a/.github/workflows/test-start-ps1.yml +++ b/.github/workflows/test-start-ps1.yml @@ -14,6 +14,10 @@ on: - 'scripts/Test-SecurityOrigin.ps1' - 'scripts/Assert-RedisPersistence.ps1' - 'scripts/Test-RedisPersistence.ps1' + - 'scripts/*infrastructure*.sh' + - 'scripts/*Infrastructure*.ps1' + - 'scripts/test-infrastructure-auto-update.py' + - 'scripts/update-env.d/*.env' - '.env.sample' - '.github/workflows/test-start-ps1.yml' pull_request: @@ -28,6 +32,10 @@ on: - 'scripts/Test-SecurityOrigin.ps1' - 'scripts/Assert-RedisPersistence.ps1' - 'scripts/Test-RedisPersistence.ps1' + - 'scripts/*infrastructure*.sh' + - 'scripts/*Infrastructure*.ps1' + - 'scripts/test-infrastructure-auto-update.py' + - 'scripts/update-env.d/*.env' - '.env.sample' - '.github/workflows/test-start-ps1.yml' workflow_dispatch: @@ -84,3 +92,6 @@ jobs: run: | powershell -ExecutionPolicy Bypass -File .\scripts\Test-SecurityOrigin.ps1 powershell -ExecutionPolicy Bypass -File .\scripts\Test-RedisPersistence.ps1 + + - name: Test Watchtower schedule registration contract + run: powershell -ExecutionPolicy Bypass -File .\scripts\Test-InfrastructureUpdateTask.ps1 diff --git a/README.md b/README.md index 89a9702..9a6bb33 100644 --- a/README.md +++ b/README.md @@ -184,9 +184,11 @@ update.sh 脚本用于自动更新 EasyAI 应用,包含以下功能: 新安装默认启用可查询进度的在线更新模式。管理员在页面发起更新后,即使主服务重启导致原请求断开,页面也会根据任务编号继续查询最终结果。存量部署如果没有 `easyai-updater` 容器,更新脚本会保留 Watchtower 模式;已经显式设置 `SYSTEM_UPDATE_MODE=legacy_watchtower` 的环境也保持该模式。只有新安装默认启用 updater。新模式只管理 `easyai-server`、`easyai-web` 和 `ws-gateway`,数据库、Redis、RabbitMQ 等基础服务仍按原部署流程维护。 -更新执行器 `easyai-updater` 由 Watchtower 单独守护,默认每 24 小时检查并更新自身。它的任务状态保存在宿主机 `data/update-state` 中,自身更新或意外重启后会继续恢复未完成任务;Watchtower 不再接管三个核心服务,因此页面任务结果不会与后台静默替换相互冲突。 +更新执行器 `easyai-updater` 由 Watchtower 默认每 24 小时检查更新,同时由宿主机定时任务每六小时直接检查更新,提供两条独立维护路径。它的任务状态保存在宿主机 `data/update-state` 中,自身更新或意外重启后会继续恢复未完成任务;Watchtower 不再接管三个核心服务,因此页面任务结果不会与后台静默替换相互冲突。 -受控更新使用仓库内的 `system-update/release-manifest.json` 作为稳定发布清单,Compose 会以只读方式挂载给 updater。发布新正式镜像时必须同步更新清单中的不可变摘要、镜像大小和版本标识;服务器执行 `update.sh` 拉取部署仓库后即可获得清单,不需要手工向运行目录复制文件。 +受控更新默认直接获取部署仓库 `main` 分支的 `system-update/release-manifest.json`,无需先在服务器执行 `git pull`。页面检测、手动更新和自动更新使用同一发布清单;任务开始后将清单保存在任务记录中,服务重启恢复时继续使用原目标。发布新正式镜像时必须同步发布清单中的不可变摘要、镜像大小和版本标识,不能只修改官方站运行版本或推送 `latest` 镜像。 + +发布源不可用时会提示暂时无法检查更新,并保留当前服务;不会用历史清单宣称已是最新。`UPDATE_MANIFEST_URL` 可配置私有发布源,留空时使用官方部署仓库。离线部署可显式设置 `UPDATE_MANIFEST_SOURCE=local`,此时读取 Compose 只读挂载的本地清单;需要自行维护该清单。旧部署中的 `UPDATE_MANIFEST_FILE` 路径覆盖不再影响默认远程模式。 新安装或已有 updater 的部署无需手工编辑 `.env`。更新脚本会从 `scripts/update-env.d/010-managed-system-update.env` 应用以下缺省值: @@ -203,7 +205,11 @@ EASYAI_DEPLOYMENT_SCHEMA_VERSION=2 `SYSTEM_UPDATE_INTERNAL_TOKEN` 由 `start.sh` / `update.sh` 自动生成并持久化,不需要手工填写。`UPDATE_HEALTH_TIMEOUT_MS` 控制每个核心服务替换后的恢复等待时间,默认 300000 毫秒。私有镜像仓库需要先执行 `docker login`;更新器与仅负责其自更新的 Watchtower 都会只读挂载 `DOCKER_CONFIG_FILE` 指向的 Docker 登录配置。切换后执行一次 `./update.sh`,确认 `easyai-updater` 以及三个核心服务均处于可用状态。 -Watchtower 固定使用 `1.22.3`。内网启用更新、事件、历史、状态和健康接口,不映射主机端口。`WATCHTOWER_API_KEY` 与 `WATCHTOWER_HTTP_API_EVENTS_TOKEN` 是两个独立随机令牌,由 Bash 和 PowerShell 安全初始化流程补齐;浏览器只访问后端,不接触这些令牌。managed 模式只扫描 updater 并定期检查自身;Watchtower 模式关闭定期扫描,只在用户或系统明确提交更新时更新核心服务,避免任务与定时扫描无法对应。 +Watchtower 默认使用 `latest`。`start.sh/update.sh` 自动安装宿主机 cron,`start.ps1/update.ps1` 自动注册 Windows 计划任务;每六小时分别对 Watchtower 和已启用的 updater 执行 `docker compose pull <服务>`、`docker compose up -d --no-deps <服务>` 并确认健康状态。只在镜像或配置有变化时重建,拉取失败不会停止该服务的现有容器。一项维护失败仍继续处理另一项,日志保留各自结果并报告失败;因此 updater 的宿主机维护不依赖 Watchtower 可用。未启用 updater 的 legacy 部署跳过 updater,不自动切换更新模式。Watchtower 的 updater 24 小时更新计划保持不变,无需额外容器。Windows 任务使用当前登录用户,以便访问其 Docker Desktop;Docker 不可用时保留失败日志,下次计划重试。 + +内网启用更新、事件、历史、状态和健康接口,不映射主机端口。`WATCHTOWER_API_KEY` 与 `WATCHTOWER_HTTP_API_EVENTS_TOKEN` 是两个独立随机令牌,由 Bash 和 PowerShell 安全初始化流程补齐;浏览器只访问后端。新安装及默认整组升级使用独立 `WATCHTOWER_SCOPE`(`easyai`),防止启动时清理其他部署的 Watchtower;定向升级和自定义监控目标保留原 scope,避免未重建的容器标签与新范围不匹配。managed 模式只扫描 updater,legacy 模式仍只在明确提交业务更新时执行原更新链路。 + +定时任务按部署目录去重,重复运行部署脚本不会重复注册。最近一次结果写入 `data/infrastructure-update/last-run.log`。设置 `INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false` 后再次执行部署脚本可移除本部署的定时任务;不改变 updater 的原有维护计划。启用时,官方镜像的旧版本标签自动迁移到 `latest`,自定义仓库或镜像摘要保持原值。旧客户需先由运维执行一次新版部署脚本,之后不用客户手工升级 Watchtower;只发布仓库不会自动把宿主机定时任务安装到未接入的机器。发布到官方 Watchtower `latest` 的镜像应先通过部署及旧接口兼容验收。 两个页面入口共用计时和步骤窗口。updater 保存阶段完成时间到 `data/update-state/jobs`;无 updater 时后端保存任务和单任务锁到 MongoDB。刷新或重新登录后按站点与账号恢复任务查询。请求超时或服务断开不会自动创建另一项更新,旧接口无法读取的阶段会明确提示。后端重启时页面保留进度,提醒在 3–5 分钟后刷新检查版本;只有确认目标版本和服务恢复才显示完成。 diff --git a/docker-compose.yml b/docker-compose.yml index 430f3f6..003461e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: #端口冲突时,只需要修改前面的端口比如修改为,3011:3010 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" ports: - "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${WEB_PORT}:3010" # read_only: true @@ -60,6 +61,7 @@ services: image: registry.cn-shanghai.aliyuncs.com/comfy-ai/comfy-server:${VERSION} # 阿里云 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" ports: - "${SERVER_HTTP_BIND_IP:-127.0.0.1}:${SERVER_HTTP_PORT}:3001" # Linux Nginx 部署默认仅回环;Windows/LAN 直连由启动脚本显式覆盖 # read_only: true @@ -166,6 +168,7 @@ services: image: registry.cn-shanghai.aliyuncs.com/easyaigc/model-conversion-worker:${MODEL_CONVERSION_WORKER_VERSION:-latest} labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" restart: unless-stopped cpus: 2 mem_limit: 4g @@ -219,6 +222,7 @@ services: image: registry.cn-shanghai.aliyuncs.com/easyaigc/wsgateway:latest # 阿里云 labels: - "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" ports: - "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${CONFIG_WS_PORT:-3002}:3002" # Linux IP/域名模式默认仅供同机 Nginx 访问 read_only: true @@ -335,6 +339,7 @@ services: image: ${RABBITMQ_IMAGE:-registry.cn-shanghai.aliyuncs.com/easyaigc/mq:latest} labels: - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" container_name: rabbitmq restart: unless-stopped environment: @@ -349,7 +354,7 @@ services: easyai: ipv4_address: 172.21.0.12 watchtower: - image: ${WATCHTOWER_IMAGE:-registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3} + image: ${WATCHTOWER_IMAGE:-registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest} volumes: - /var/run/docker.sock:/var/run/docker.sock - ${DOCKER_CONFIG_FILE:-/root/.docker/config.json}:/config.json:ro @@ -358,6 +363,7 @@ services: environment: - WATCHTOWER_POLL_INTERVAL=${SYSTEM_UPDATER_UPDATE_INTERVAL_SECONDS:-86400} - WATCHTOWER_CLEANUP=${WATCHTOWER_CLEANUP:-false} + - WATCHTOWER_SCOPE=${WATCHTOWER_SCOPE:-} - WATCHTOWER_HTTP_API_HOST=0.0.0.0 - WATCHTOWER_HTTP_API_ENDPOINTS=health,update,metrics,events,history - WATCHTOWER_HTTP_API_TOKEN=${WATCHTOWER_API_KEY} @@ -366,6 +372,7 @@ services: restart: always labels: - "com.centurylinklabs.watchtower.enable=false" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" # ports: # - "${WATCHTOWER_PORT}:8080" read_only: true @@ -385,6 +392,7 @@ services: - managed-update labels: - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" restart: unless-stopped read_only: true cap_drop: @@ -399,6 +407,8 @@ services: - DOCKER_CONFIG_FILE=/run/secrets/docker-config.json - UPDATE_STATE_DIR=/var/lib/easyai-updater - UPDATE_MANIFEST_FILE=/etc/easyai-updater/release-manifest.json + - UPDATE_MANIFEST_SOURCE=${UPDATE_MANIFEST_SOURCE:-remote} + - UPDATE_MANIFEST_URL=${UPDATE_MANIFEST_URL:-} volumes: - /var/run/docker.sock:/var/run/docker.sock - ${DOCKER_CONFIG_FILE:-/root/.docker/config.json}:/run/secrets/docker-config.json:ro @@ -482,6 +492,7 @@ services: # 不映射宿主机端口,仅允许 Docker 内网中的 easyai-server 访问 labels: - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" volumes: - ./sandbox-data:/app/data - ./local_packages:/app/local_packages @@ -554,6 +565,7 @@ services: image: registry.cn-shanghai.aliyuncs.com/easyaigc/agent-governance:${ASG_VERSION:-latest} labels: - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" ports: - "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${ASG_PORT:-3003}:3003" networks: @@ -585,6 +597,7 @@ services: image: registry.cn-shanghai.aliyuncs.com/easyaigc/agent-memory:${AMS_VERSION:-latest} labels: - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}" ports: - "${EASYAI_INFRA_BIND_IP:-127.0.0.1}:${AMS_PORT:-3004}:3004" environment: diff --git a/scripts/Initialize-InfrastructureUpdateEnv.ps1 b/scripts/Initialize-InfrastructureUpdateEnv.ps1 new file mode 100644 index 0000000..a0e98bf --- /dev/null +++ b/scripts/Initialize-InfrastructureUpdateEnv.ps1 @@ -0,0 +1,20 @@ +param([Parameter(Mandatory=$true)][string]$ProjectRoot) +$ErrorActionPreference = 'Stop' +. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1') +$path = Join-Path $ProjectRoot '.env' +$content = [System.IO.File]::ReadAllText($path) +if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return } +$image = Get-EnvValue $content 'WATCHTOWER_IMAGE' +if ([string]::IsNullOrWhiteSpace($image) -or $image -match '^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:\d+\.\d+\.\d+$') { + $content = Set-EnvValue $content 'WATCHTOWER_IMAGE' 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' + [System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false)) +} + +# Preserve old scope during a targeted update; unchanged containers keep their labels. +$image = Get-EnvValue $content 'WATCHTOWER_IMAGE' +$target = Get-EnvValue $content 'WATCHTOWER_UPDATE_TARGET' +$scope = Get-EnvValue $content 'WATCHTOWER_SCOPE' +if ([string]::IsNullOrWhiteSpace($env:UPDATE_SERVICES) -and [string]::IsNullOrWhiteSpace($scope) -and $image -eq 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' -and $target -in @('', 'easyai-updater', 'easyai-server easyai-wsgateway easyai-web')) { + $content = Set-EnvValue $content 'WATCHTOWER_SCOPE' 'easyai' + [System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false)) +} diff --git a/scripts/Install-InfrastructureUpdateTask.ps1 b/scripts/Install-InfrastructureUpdateTask.ps1 new file mode 100644 index 0000000..1c83c00 --- /dev/null +++ b/scripts/Install-InfrastructureUpdateTask.ps1 @@ -0,0 +1,30 @@ +param( + [Parameter(Mandatory=$true)][string]$ProjectRoot, + [switch]$DryRun +) +$ErrorActionPreference = 'Stop' +$ProjectRoot = (Resolve-Path $ProjectRoot).Path +. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1') +$enabled = (Get-EnvValue ([System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env'))) 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -ne 'false' +$hash = [System.Security.Cryptography.SHA256]::Create() +try { $id = ([System.BitConverter]::ToString($hash.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($ProjectRoot)))).Replace('-', '').Substring(0, 12) } +finally { $hash.Dispose() } +$name = "EasyAI-Watchtower-$id" +if ($DryRun -and -not $enabled) { Write-Host "更新组件定时任务(dry-run):已关闭"; return } +if ($DryRun) { Write-Host "Watchtower 和 updater 定时任务(dry-run):$name,每六小时检查一次"; return } +if (-not $enabled) { + $existing = Get-ScheduledTask -TaskName $name -ErrorAction SilentlyContinue + if ($existing) { Unregister-ScheduledTask -TaskName $name -Confirm:$false } + return +} +$executable = (Get-Process -Id $PID).Path +$script = Join-Path $ProjectRoot 'scripts/Update-Infrastructure.ps1' +$action = New-ScheduledTaskAction -Execute $executable -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$script`" -ProjectRoot `"$ProjectRoot`"" +$triggers = @('00:17', '06:17', '12:17', '18:17') | ForEach-Object { New-ScheduledTaskTrigger -Daily -At $_ } +# Docker Desktop belongs to the logged-in user, not the SYSTEM account. +$user = [System.Environment]::UserName +if ($env:USERDOMAIN) { $user = "$env:USERDOMAIN\$user" } +$principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 2) +Register-ScheduledTask -TaskName $name -Action $action -Trigger $triggers -Principal $principal -Settings $settings -Force | Out-Null +Write-Host '已配置 Watchtower 和 updater 定时任务:每六小时检查一次(Docker Desktop 可用时执行)' diff --git a/scripts/Test-InfrastructureUpdateTask.ps1 b/scripts/Test-InfrastructureUpdateTask.ps1 new file mode 100644 index 0000000..28a131e --- /dev/null +++ b/scripts/Test-InfrastructureUpdateTask.ps1 @@ -0,0 +1,31 @@ +# Test scheduler registration without changing the host's real scheduled tasks. +$ErrorActionPreference = 'Stop' +$started = Get-Date +$root = Join-Path ([System.IO.Path]::GetTempPath()) ('easyai-task-' + [guid]::NewGuid().ToString('N')) +[System.IO.Directory]::CreateDirectory((Join-Path $root 'scripts')) | Out-Null +try { + Copy-Item (Join-Path $PSScriptRoot 'Initialize-SecurityEnv.ps1') (Join-Path $root 'scripts') + Copy-Item (Join-Path $PSScriptRoot 'Install-InfrastructureUpdateTask.ps1') (Join-Path $root 'scripts') + $global:infraTaskTestTasks = @{ 'Unrelated-Backup' = @{ keep = $true } } + function New-ScheduledTaskAction { param($Execute, $Argument); return @{Execute=$Execute; Argument=$Argument} } + function New-ScheduledTaskTrigger { param([switch]$Daily, $At); return @{Daily=$Daily; At=$At} } + function New-ScheduledTaskPrincipal { param($UserId, $LogonType, $RunLevel); return @{User=$UserId; Logon=$LogonType; Level=$RunLevel} } + function New-ScheduledTaskSettingsSet { param([switch]$StartWhenAvailable, $MultipleInstances, $ExecutionTimeLimit); return @{Concurrent=$MultipleInstances; Available=$StartWhenAvailable} } + function Register-ScheduledTask { param($TaskName, $Action, $Trigger, $Principal, $Settings, [switch]$Force); $global:infraTaskTestTasks[$TaskName]=@{Action=$Action; Trigger=$Trigger; Principal=$Principal; Settings=$Settings}; return @{} } + function Get-ScheduledTask { param($TaskName, $ErrorAction); return $global:infraTaskTestTasks[$TaskName] } + function Unregister-ScheduledTask { param($TaskName, $Confirm); $global:infraTaskTestTasks.Remove($TaskName) } + $envFile = Join-Path $root '.env' + [System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true') + & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root + & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root + if ($global:infraTaskTestTasks.Count -ne 2) { throw 'Repeated install duplicated tasks or deleted unrelated tasks' } + $key = @($global:infraTaskTestTasks.Keys | Where-Object { $_ -like 'EasyAI-Watchtower-*' })[0] + $task = $global:infraTaskTestTasks[$key] + if ($task.Trigger.Count -ne 4 -or $task.Settings.Concurrent -ne 'IgnoreNew') { throw 'Schedule or concurrency policy changed' } + if ($task.Principal.Logon -ne 'Interactive') { throw 'Task would not use the Docker Desktop user' } + if ($task.Action.Argument -notlike '*Update-Infrastructure.ps1*' -or $task.Action.Argument -notlike '*-NonInteractive*') { throw 'Incorrect task command' } + [System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false') + & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root + if ($global:infraTaskTestTasks.Count -ne 1 -or -not $global:infraTaskTestTasks.ContainsKey('Unrelated-Backup')) { throw 'Disable removed an unrelated task' } + Write-Host "PowerShell scheduler contract passed in $([math]::Round(((Get-Date)-$started).TotalSeconds,2)) seconds" +} finally { Remove-Item -Recurse -Force $root; Remove-Variable infraTaskTestTasks -Scope Global -ErrorAction SilentlyContinue } diff --git a/scripts/Update-Infrastructure.ps1 b/scripts/Update-Infrastructure.ps1 new file mode 100644 index 0000000..bd56628 --- /dev/null +++ b/scripts/Update-Infrastructure.ps1 @@ -0,0 +1,74 @@ +param([string]$ProjectRoot = (Split-Path $PSScriptRoot -Parent)) +$ErrorActionPreference = 'Stop' +Set-Location $ProjectRoot +. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1') +$content = [System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env')) +if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return } +$state = Join-Path $ProjectRoot 'data/infrastructure-update' +[System.IO.Directory]::CreateDirectory($state) | Out-Null +try { $lock = [System.IO.File]::Open((Join-Path $state 'task.lock'), 'OpenOrCreate', 'ReadWrite', 'None') } +catch [System.IO.IOException] { return } +$log = Join-Path $state 'last-run.log' +$previousConfig = $env:DOCKER_CONFIG +try { + [System.IO.File]::WriteAllText($log, "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') 检查 Watchtower 和 updater 更新`n") + $configFile = Get-EnvValue $content 'DOCKER_CONFIG_FILE' + if (-not [string]::IsNullOrWhiteSpace($configFile) -and (Test-Path $configFile -PathType Leaf)) { + $env:DOCKER_CONFIG = Split-Path $configFile -Parent + } + & docker compose version *> $null + $v2 = $LASTEXITCODE -eq 0 + function Invoke-InfrastructureCompose { + param([string[]]$ComposeArguments) + if ($v2) { $output = & docker compose @ComposeArguments 2>&1 } + else { $output = & docker-compose @ComposeArguments 2>&1 } + $code = $LASTEXITCODE + $output | Out-File $log -Append -Encoding utf8 + if ($code -ne 0) { throw '组件更新命令未成功,现有容器状态需查看日志' } + } + if ($v2) { $available = @(& docker compose config --services) } + else { $available = @(& docker-compose config --services) } + if ($LASTEXITCODE -ne 0 -or $available -notcontains 'watchtower') { throw '无法读取 Watchtower 部署配置' } + $failed = @() + foreach ($service in @('watchtower', 'easyai-updater')) { + # Do not enable updater in profile-disabled legacy deployments. + if ($available -notcontains $service) { + "$service 未启用,跳过维护" | Out-File $log -Append -Encoding utf8 + continue + } + try { + "检查 $service" | Out-File $log -Append -Encoding utf8 + Invoke-InfrastructureCompose -ComposeArguments @('pull', $service) + Invoke-InfrastructureCompose -ComposeArguments @('up', '-d', '--no-deps', $service) + $healthy = $false + for ($attempt=0; $attempt -lt 12; $attempt++) { + if ($v2) { $containerId = & docker compose ps -q $service } + else { $containerId = & docker-compose ps -q $service } + if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" } + if ($containerId) { + $status = & docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $containerId + if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" } + if ($status -eq 'true healthy') { $healthy = $true; break } + if ($status -eq 'true none') { + if ($service -eq 'watchtower') { + & docker exec $containerId /watchtower --health-check *> $null + } else { + & docker exec $containerId node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))" *> $null + } + if ($LASTEXITCODE -eq 0) { $healthy = $true; break } + } + } + Start-Sleep -Seconds 5 + } + if (-not $healthy) { throw "$service 暂未恢复可用" } + "$service 已恢复可用" | Out-File $log -Append -Encoding utf8 + } catch { + "$service 本次维护未成功,下次定时任务将继续检查:$($_.Exception.Message)" | Out-File $log -Append -Encoding utf8 + $failed += $service + } + } + if ($failed.Count -gt 0) { throw "更新组件维护未全部成功:$($failed -join ', ')" } +} finally { + $env:DOCKER_CONFIG = $previousConfig + $lock.Dispose() +} diff --git a/scripts/init-infrastructure-update-env.sh b/scripts/init-infrastructure-update-env.sh new file mode 100755 index 0000000..583510f --- /dev/null +++ b/scripts/init-infrastructure-update-env.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail +PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +. "$PROJECT_ROOT/scripts/init-security-env.sh" +[ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0 +image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)" +if [ -z "$image" ] || [[ "$image" =~ ^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest +fi +# Scope changes require the selected containers to receive matching labels. +# A targeted update must not strand unchanged containers outside the new scope. +image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)" +target="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_UPDATE_TARGET)" +if [ -z "${UPDATE_SERVICES:-}" ] && [ -z "$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE)" ] && [ "$image" = registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest ]; then + case "$target" in + ''|'easyai-updater'|'easyai-server easyai-wsgateway easyai-web') + write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE easyai + ;; + esac +fi diff --git a/scripts/install-infrastructure-update-task.sh b/scripts/install-infrastructure-update-task.sh new file mode 100755 index 0000000..811433b --- /dev/null +++ b/scripts/install-infrastructure-update-task.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -euo pipefail +PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)" +. "$PROJECT_ROOT/scripts/init-security-env.sh" +# A path is shell-quoted and percent-escaped before embedding it in crontab. +[[ "$PROJECT_ROOT" != *$'\n'* ]] || { echo '部署目录不能包含换行'; exit 1; } +task_id="$(printf '%s' "$PROJECT_ROOT" | cksum | awk '{print $1}')" +marker="# easyai-infrastructure-update-$task_id" +job="$(printf '%q' "$PROJECT_ROOT/scripts/update-infrastructure.sh")" +job="${job//%/\\%}" +entry="17 */6 * * * PATH=/usr/local/bin:/opt/homebrew/bin:/usr/sbin:/usr/bin:/sbin:/bin /bin/bash $job >/dev/null 2>&1 $marker" +if [ "${2:-}" = --dry-run ]; then + if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then echo "更新组件定时任务(dry-run):已关闭"; exit 0; fi + printf 'Watchtower 和 updater 定时任务(dry-run):%s\n' "$entry" + exit 0 +fi +cron() { + if [ "$(id -u)" -eq 0 ]; then crontab "$@"; else sudo crontab "$@"; fi +} +if ! command -v crontab >/dev/null; then + [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0 + if command -v apt-get >/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y cron + else + echo '缺少 crontab,无法安装更新组件定时任务' >&2 + exit 1 + fi +fi +if [ "$(uname -s)" = Linux ]; then + if [ -d /run/systemd/system ]; then + sudo systemctl enable --now cron + elif command -v service >/dev/null; then + sudo service cron start + fi +fi +error_file="$(mktemp)" +trap 'rm -f "$error_file"' EXIT +if ! current="$(LC_ALL=C cron -l 2>"$error_file")"; then + if ! grep -qi 'no crontab for' "$error_file"; then + echo '无法读取现有定时任务,未修改任务配置' >&2 + exit 1 + fi + current="" +fi +# Keep all unrelated tasks and replace this deployment's single entry. +next="$(printf '%s\n' "$current" | awk -v marker="$marker" 'substr($0, length($0)-length(marker)+1) != marker')" +if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ]; then + next="${next:+$next$'\n'}$entry" +fi +printf '%s\n' "$next" | cron - +if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then + echo '已移除本部署的更新组件定时任务' +else + echo '已配置 Watchtower 和 updater 定时任务:每六小时检查一次' +fi diff --git a/scripts/test-deployment-public-url.sh b/scripts/test-deployment-public-url.sh index b3404f7..c5b5d2e 100755 --- a/scripts/test-deployment-public-url.sh +++ b/scripts/test-deployment-public-url.sh @@ -20,6 +20,9 @@ mkdir -p "$TMP_DIR/scripts" mkdir -p "$TMP_DIR/system-update" cp \ "$REPO_ROOT/scripts/init-security-env.sh" \ + "$REPO_ROOT/scripts/init-infrastructure-update-env.sh" \ + "$REPO_ROOT/scripts/install-infrastructure-update-task.sh" \ + "$REPO_ROOT/scripts/update-infrastructure.sh" \ "$REPO_ROOT/scripts/init-security-origin.sh" \ "$REPO_ROOT/scripts/init-public-api-base-url.sh" \ "$REPO_ROOT/scripts/init-server-http-bind-ip.sh" \ @@ -186,6 +189,8 @@ assert str(server_env.get("EASYAI_DEPLOYMENT_SCHEMA_VERSION")) == "2" assert str(updater_env.get("UPDATE_HEALTH_TIMEOUT_MS")) == "300000" assert updater_env.get("DOCKER_CONFIG_FILE") == "/run/secrets/docker-config.json" assert updater_env.get("UPDATE_MANIFEST_FILE") == "/etc/easyai-updater/release-manifest.json" +assert updater_env.get("UPDATE_MANIFEST_SOURCE") == "remote" +assert updater_env.get("UPDATE_MANIFEST_URL") == "" assert updater.get("labels", {}).get("com.centurylinklabs.watchtower.enable") == "true" assert watchtower.get("environment", {}).get("WATCHTOWER_POLL_INTERVAL") == "86400" assert watchtower.get("environment", {}).get("WATCHTOWER_CLEANUP") == "true" diff --git a/scripts/test-infrastructure-auto-update.py b/scripts/test-infrastructure-auto-update.py new file mode 100644 index 0000000..4d54a29 --- /dev/null +++ b/scripts/test-infrastructure-auto-update.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Exercise real entry scripts in temporary projects; never edits host schedulers.""" +from pathlib import Path +import json,os,subprocess,tempfile,time,shutil +REPO=Path(__file__).resolve().parents[1] +started=time.monotonic() +with tempfile.TemporaryDirectory(prefix='easyai infrastructure % ') as tmp: + root=Path(tmp); project=root/'project space';(project/'scripts').mkdir(parents=True) + for name in ['init-security-env.sh','Initialize-SecurityEnv.ps1','init-infrastructure-update-env.sh','Initialize-InfrastructureUpdateEnv.ps1','install-infrastructure-update-task.sh','Install-InfrastructureUpdateTask.ps1','update-infrastructure.sh','Update-Infrastructure.ps1']: + shutil.copy2(REPO/'scripts'/name,project/'scripts'/name) + bins=root/'bin';bins.mkdir();calls=root/'calls.jsonl';cron=root/'crontab';cron.write_text('5 2 * * * existing-backup\n# existing comment\n') + environment={**os.environ,'PATH':str(bins)+os.pathsep+os.environ['PATH'],'INFRA_TEST_CALLS':str(calls),'INFRA_TEST_CRON':str(cron)} + def stub(name,body): + p=bins/name;p.write_text(body);p.chmod(0o755) + stub('sudo','#!/bin/sh\nexec "$@"\n') + stub('service','#!/bin/sh\nexit 0\n') + stub('systemctl','#!/bin/sh\nexit 0\n') + stub('crontab','''#!/usr/bin/env python3 +import sys,os +from pathlib import Path +p=Path(os.environ['INFRA_TEST_CRON']) +if os.environ.get('INFRA_TEST_CRON_ERROR'): + print('permission denied',file=sys.stderr);sys.exit(1) +if sys.argv[1]=='-l':print(p.read_text(),end='') +else:p.write_text(sys.stdin.read()) +''') + stub('docker','''#!/usr/bin/env python3 +import sys,os,json,time +from pathlib import Path +args=sys.argv[1:] +with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(args)+'\\n') +if args[:1]==['--config']:args=args[2:] +if args==['compose','version']:sys.exit(1 if os.environ.get('INFRA_TEST_V1') else 0) +if args[:3]==['compose','ps','-q']:print(args[3]+'-test-id') +if args[:1]==['inspect']:print('true none' if os.environ.get('INFRA_TEST_NO_HEALTH') else 'true healthy') +if args==['compose','config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo') +if args[:2]==['compose','pull']: + time.sleep(float(os.environ.get('INFRA_TEST_SLEEP','0'))) + if os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',args[2]]:sys.exit(1) +''') + stub('docker-compose','''#!/usr/bin/env python3 +import sys,os,json +from pathlib import Path +with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(['v1']+sys.argv[1:])+'\\n') +if sys.argv[1:3]==['ps','-q']:print(sys.argv[3]+'-test-id') +if sys.argv[1:]==['config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo') +if sys.argv[1:2]==['pull'] and os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',sys.argv[2]]:sys.exit(1) +''') + def run(args,extra=None): + return subprocess.run(args,cwd=project,env={**environment,**(extra or {})},capture_output=True,text=True) + def task():return ['bash',str(project/'scripts/update-infrastructure.sh')] + def recorded():return [json.loads(line) for line in calls.read_text().splitlines()] + for shell in ['bash','pwsh']: + for image,enabled,expected in [('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','true','latest'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.13.1','true','latest'),('customer.invalid/watcher:1.22.3','true','1.22.3'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','false','1.22.3')]: + (project/'.env').write_text(f'WATCHTOWER_IMAGE={image}\nINFRASTRUCTURE_AUTO_UPDATE_ENABLED={enabled}\nKEEP=original\n') + args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)] + p=run(args);assert p.returncode==0,p.stderr + content=(project/'.env').read_text();assert f'WATCHTOWER_IMAGE={image.rsplit(":",1)[0]}:{expected}' in content,content + assert 'KEEP=original' in content + assert run(args).returncode==0 and (project/'.env').read_text()==content + for shell in ['bash','pwsh']: + (project/'.env').write_text('WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest\nWATCHTOWER_UPDATE_TARGET=easyai-updater\n') + args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)] + assert run(args,{'UPDATE_SERVICES':'easyai-server'}).returncode==0 + assert 'WATCHTOWER_SCOPE=' not in (project/'.env').read_text() + assert run(args,{'UPDATE_SERVICES':''}).returncode==0 + assert 'WATCHTOWER_SCOPE=easyai' in (project/'.env').read_text() + (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n') + install=['bash',str(project/'scripts/install-infrastructure-update-task.sh'),str(project)] + assert run(install).returncode==0 + first=cron.read_text();assert 'existing-backup' in first and first.count('# easyai-infrastructure-update-')==1 + assert '\\%' in first and 'project\\ space' in first,first + assert run(install).returncode==0 and cron.read_text()==first + assert run(install,{'INFRA_TEST_CRON_ERROR':'1'}).returncode!=0 and cron.read_text()==first + (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false\n') + assert run(install).returncode==0 and 'easyai-infrastructure-update' not in cron.read_text() and 'existing-backup' in cron.read_text() + (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n') + calls.write_text('');assert run(task()).returncode==0 + assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded() + assert ['compose','pull','easyai-updater'] in recorded() and ['compose','up','-d','--no-deps','easyai-updater'] in recorded() + assert not any('mongo' in args for args in recorded()) + calls.write_text('');assert run(task(),{'INFRA_TEST_PULL_FAIL':'1'}).returncode!=0 + assert not any('up' in args for args in recorded());assert not (project/'data/infrastructure-update/lock').exists() + calls.write_text('');p=subprocess.Popen(task(),cwd=project,env={**environment,'INFRA_TEST_SLEEP':'1'},stdout=subprocess.PIPE,stderr=subprocess.PIPE) + for _ in range(100): + if (project/'data/infrastructure-update/lock/pid').exists():break + time.sleep(.01) + assert run(task()).returncode==0 + assert p.wait(timeout=5)==0 and sum('pull' in args for args in recorded())==2 + calls.write_text('');assert run(task(),{'INFRA_TEST_V1':'1'}).returncode==0 + assert ['v1','up','-d','--no-deps','watchtower'] in recorded() and ['v1','up','-d','--no-deps','easyai-updater'] in recorded() + calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)]) + assert p.returncode==0,p.stderr + assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded(),recorded() + calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)],{'INFRA_TEST_PULL_FAIL':'1'}) + assert p.returncode!=0 and not any('up' in args for args in recorded()) + for shell in ['bash','pwsh']: + args=task() if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)] + for transport in ['', '1']: + kind='v1' if transport else 'compose' + for failed,other in [('watchtower','easyai-updater'),('easyai-updater','watchtower')]: + calls.write_text('');p=run(args,{'INFRA_TEST_PULL_FAIL':failed,'INFRA_TEST_V1':transport}) + assert p.returncode!=0,(shell,p.stderr) + assert [kind,'pull',failed] in recorded() and [kind,'pull',other] in recorded() + assert [kind,'up','-d','--no-deps',failed] not in recorded() + assert [kind,'up','-d','--no-deps',other] in recorded(),(shell,recorded()) + calls.write_text('');p=run(args,{'INFRA_TEST_LEGACY':'1','INFRA_TEST_V1':transport}) + assert p.returncode==0 and not any('easyai-updater' in a for a in recorded()),(shell,p.stderr,recorded()) + calls.write_text('');p=run(args,{'INFRA_TEST_NO_HEALTH':'1'}) + assert p.returncode==0,(shell,p.stderr) + assert any(a[:3]==['exec','easyai-updater-test-id','node'] for a in recorded()) +print(json.dumps({'passed':True,'cases':['Bash/PowerShell default migration and explicit choices','cron registration deduplication and existing task preservation','cron read failure leaves jobs untouched','Bash/PowerShell independent component maintenance and partial failure reporting','legacy profile skips updater','concurrent task excluded','Compose v1 compatibility','only Watchtower and updater updated','service health probes'],'durationSeconds':round(time.monotonic()-started,3)})) diff --git a/scripts/test-model-conversion-update.py b/scripts/test-model-conversion-update.py index a53df85..073f7f8 100644 --- a/scripts/test-model-conversion-update.py +++ b/scripts/test-model-conversion-update.py @@ -34,6 +34,8 @@ def fixture(base): shutil.copytree(path, root / name) else: shutil.copy2(path, root / name) + (root / 'scripts/install-infrastructure-update-task.sh').write_text('#!/bin/sh\nexit 0\n') + (root / 'scripts/Install-InfrastructureUpdateTask.ps1').write_text('# Scheduler transport is verified separately.\n') (root / 'data').mkdir() (root / '.env').write_text('SYSTEM_UPDATE_MODE=legacy_watchtower\nNUXT_PUBLIC_BASE_APIURL=http://127.0.0.1:3001\nCONFIG_JWT_SECRET=existing-jwt-secret-that-is-long-enough\nWS_AUTH_WS_TICKET_SECRET=existing-ticket-secret-that-is-long-enough\n') (root / '.gitignore').write_text('.env\n*.log\n') diff --git a/scripts/test-update-env-migrations.sh b/scripts/test-update-env-migrations.sh index 310e969..089b3e1 100755 --- a/scripts/test-update-env-migrations.sh +++ b/scripts/test-update-env-migrations.sh @@ -26,7 +26,7 @@ grep -qx 'COMPOSE_PROFILES=customer-profile' "$LEGACY_ENV_FILE" grep -qx 'WATCHTOWER_CLEANUP=' "$LEGACY_ENV_FILE" grep -qx 'WATCHTOWER_HTTP_API_PERIODIC_POLLS=false' "$LEGACY_ENV_FILE" grep -qx 'WATCHTOWER_UPDATE_TARGET=easyai-server easyai-wsgateway easyai-web' "$LEGACY_ENV_FILE" -grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3' "$LEGACY_ENV_FILE" +grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' "$LEGACY_ENV_FILE" grep -qx 'MODEL_CONVERSION_WORKER_VERSION=latest' "$LEGACY_ENV_FILE" grep -qx 'MODEL_CONVERSION_HUNYUAN_ENABLED=false' "$LEGACY_ENV_FILE" cp "$LEGACY_ENV_FILE" "$TMP_DIR/legacy-first-run.env" diff --git a/scripts/update-env.d/020-watchtower-progress.env b/scripts/update-env.d/020-watchtower-progress.env index 5e7482e..40b4b84 100644 --- a/scripts/update-env.d/020-watchtower-progress.env +++ b/scripts/update-env.d/020-watchtower-progress.env @@ -1,2 +1,2 @@ # 两种更新模式均启用内网接口;安全令牌由安全初始化脚本生成。 -WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3 +WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest diff --git a/scripts/update-env.d/023-infrastructure-auto-update.env b/scripts/update-env.d/023-infrastructure-auto-update.env new file mode 100644 index 0000000..cdd4074 --- /dev/null +++ b/scripts/update-env.d/023-infrastructure-auto-update.env @@ -0,0 +1,2 @@ +# 宿主机定时维护更新组件,保留客户明确关闭的选择。 +INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true diff --git a/scripts/update-infrastructure.sh b/scripts/update-infrastructure.sh new file mode 100755 index 0000000..33b13c1 --- /dev/null +++ b/scripts/update-infrastructure.sh @@ -0,0 +1,81 @@ +#!/bin/bash +set -euo pipefail +PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" +cd "$PROJECT_ROOT" +. ./scripts/init-security-env.sh +[ "$(read_env_value .env INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0 +state="$PROJECT_ROOT/data/infrastructure-update" +mkdir -p "$state" +lock="$state/lock" +if ! mkdir "$lock" 2>/dev/null; then + pid="$(cat "$lock/pid" 2>/dev/null || true)" + if [[ "$pid" =~ ^[0-9]+$ ]] && ! kill -0 "$pid" 2>/dev/null; then + rm -f "$lock/pid" + rmdir "$lock" 2>/dev/null || exit 0 + mkdir "$lock" 2>/dev/null || exit 0 + else + exit 0 + fi +fi +printf '%s\n' "$$" > "$lock/pid" +trap 'rm -f "$lock/pid"; rmdir "$lock" 2>/dev/null || true' EXIT +exec > "$state/last-run.log" 2>&1 +printf '%s 检查 Watchtower 和 updater 更新\n' "$(date '+%Y-%m-%d %H:%M:%S')" +registry_config="$(read_env_value .env DOCKER_CONFIG_FILE)" +docker_options=() +if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then + docker_options=(--config "$(dirname "$registry_config")") +fi +if docker "${docker_options[@]}" compose version >/dev/null 2>&1; then + compose() { docker "${docker_options[@]}" compose "$@"; } +else + compose() { + if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then + DOCKER_CONFIG="$(dirname "$registry_config")" docker-compose "$@" + else + docker-compose "$@" + fi + } +fi +available="$(compose config --services)" +printf '%s\n' "$available" | grep -qx watchtower || { echo '未配置 Watchtower'; exit 1; } +maintain_service() { + local service="$1" container_id status attempt + echo "检查 $service" + # A failed service must not block the other update channel. + compose pull "$service" || return 1 + compose up -d --no-deps "$service" || return 1 + for ((attempt=0; attempt<12; attempt++)); do + container_id="$(compose ps -q "$service")" || return 1 + if [ -n "$container_id" ]; then + status="$(docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")" || return 1 + if [ "$status" = 'true healthy' ]; then + echo "$service 已恢复可用" + return 0 + fi + if [ "$status" = 'true none' ]; then + if [ "$service" = watchtower ]; then + if docker exec "$container_id" /watchtower --health-check; then return 0; fi + else + if docker exec "$container_id" node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))"; then return 0; fi + fi + fi + fi + sleep 5 + done + echo "$service 暂未恢复可用" >&2 + return 1 +} +failed=0 +for service in watchtower easyai-updater; do + # config --services respects active profiles: legacy deployments stay legacy. + if ! printf '%s\n' "$available" | grep -qx "$service"; then + echo "$service 未启用,跳过维护" + continue + fi + if ! maintain_service "$service"; then + echo "$service 本次维护未成功,下次定时任务将继续检查" >&2 + failed=1 + fi +done +exit "$failed" diff --git a/start.ps1 b/start.ps1 index 92eb5bc..acad769 100644 --- a/start.ps1 +++ b/start.ps1 @@ -429,6 +429,7 @@ function Main { . (Join-Path $script:Root "scripts\Initialize-SecurityEnv.ps1") Initialize-SecurityEnv -Path (Join-Path $script:Root ".env") -Mode $script:SecurityEnvMode + & (Join-Path $script:Root "scripts/Initialize-InfrastructureUpdateEnv.ps1") -ProjectRoot $script:Root . (Join-Path $script:Root "scripts\Initialize-PublicApiBaseUrl.ps1") $publicUrlMode = if ($script:SkipDeployQuestions) { "upgrade" } else { "configure" } @@ -442,9 +443,11 @@ function Main { if ($script:DeployDryRun) { Warn "dry-run mode: skip docker and services" + & (Join-Path $script:Root "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $script:Root -DryRun } else { Test-Docker Start-Services + & (Join-Path $script:Root "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $script:Root } $ip = if ($script:DeployIP) { $script:DeployIP } else { "127.0.0.1" } diff --git a/start.sh b/start.sh index 0babdd5..9e2cf84 100755 --- a/start.sh +++ b/start.sh @@ -648,6 +648,7 @@ main() { # shellcheck source=scripts/init-security-env.sh . ./scripts/init-security-env.sh init_security_env .env "$SECURITY_ENV_MODE" + bash ./scripts/init-infrastructure-update-env.sh "$PWD" # 在拉取镜像或重建服务前完成公开地址迁移;非法或无法推导时立即终止。 # shellcheck source=scripts/init-public-api-base-url.sh @@ -666,9 +667,11 @@ main() { echo "" echo "⚠️ dry-run 模式:跳过 Docker 安装和服务启动" echo " 配置文件已生成,可直接运行 ./start.sh 完成部署" + bash ./scripts/install-infrastructure-update-task.sh "$PWD" --dry-run else install_docker start_services + bash ./scripts/install-infrastructure-update-task.sh "$PWD" configure_http_nginx_proxy export DEPLOY_DOMAIN run_https_setup diff --git a/system-update/releases/infrastructure-auto-update-20261008/acceptance.md b/system-update/releases/infrastructure-auto-update-20261008/acceptance.md new file mode 100644 index 0000000..65ac7d1 --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/acceptance.md @@ -0,0 +1,65 @@ +# updater 双保险维护验收 + +日期:2026-10-08。代码位于 `/Users/wangbo/projects/easyai`。本记录补充前序 Watchtower 宿主机维护验收,描述本轮新增的 updater 独立定时更新通道。 + +## 结果 + +宿主机定时任务每六小时分别维护 Watchtower 和已启用的 updater;原 Watchtower 每 24 小时维护 updater 的机制保留。无需新增容器或再注册另一条宿主机任务。 + +每项维护分别拉取、按需重建和验证健康状态。某项失败仍继续另一项,同时记录失败并返回非零结果。不会因 Watchtower 不可用而跳过 updater。旧部署未启用 updater 时保持原更新模式。 + +验收时尚未提交或推送源码;本轮真实运行结果不代表已部署至线上。本轮没有实际注册本机 cron 或 Windows 计划任务。旧机器需由运维执行一次新版部署脚本接入;发布源码本身不能给机器安装宿主机任务。 + +## 修改前真实复现 + +独立 Compose 项目 `easyai-updater-insurance-20261008`,Watchtower 1.22.3、updater 1.1.0,本机 registry 发布 updater 1.2.0。主动停止本次自建的 Watchtower,并把该项目 Watchtower 镜像设置为不存在的标签,模拟 Watchtower 恢复失败。 + +分别运行原 Bash 和 PowerShell 定时入口:均在 Watchtower 拉取失败时退出;updater 继续运行旧版本 1.1.0,容器及镜像均未变化。 + +证据:`before.json`、`before-bash.log`、`before-powershell.log`,原入口快照保存在 `before-scripts/`。这说明原定时维护只负责 Watchtower,不能在其失效时直接维护 updater。 + +## 本轮变更及影响范围 + +- 修改 `scripts/update-infrastructure.sh` 和 `scripts/Update-Infrastructure.ps1`:独立处理两个组件,失败分别记录,最后汇总结果;保持部署目录、Docker 登录配置、并发锁及只重建指定服务的行为。 +- 修改 Bash/PowerShell 任务安装器的提示文案,保留原任务标识、每六小时周期、去重及关闭方式,不重复注册任务。 +- 更新 `.env.sample` 和 README,说明 updater 两条维护路径。 +- 扩展 `scripts/test-infrastructure-auto-update.py`,覆盖两种入口、两种 Compose 命令路径、独立失败、legacy 跳过及健康检查。 + +主要调用方仍为 `start.sh/update.sh/start.ps1/update.ps1` 安装的定时任务。没有修改业务自动更新开关,也没有改动 updater、后端、前端业务代码及已有任务存储格式。此前工作区中其他未提交改动均保留。 + +## 实际运行结果 + +| 场景 | 实际结果 | +| --- | --- | +| Watchtower 已停止且拉取失败,Bash 定时入口 | updater 实际升级 1.1.0 → 1.2.0;Watchtower 仍停止,证明该升级不依赖它;返回失败以保留 Watchtower 故障结果 | +| 相同条件,PowerShell 定时入口 | updater 实际升级 1.1.0 → 1.2.0;同样独立执行并报告部分失败 | +| updater 历史任务 | Bash 独立更新后,实际任务 API 仍能读取原有成功任务及其消息 | +| updater 拉取失败 | updater 容器未停止或重建;Watchtower 仍恢复健康;整体报告失败 | +| 正常维护及镜像无变化 | 两组件均健康;镜像和配置无变化时容器 ID 不变 | +| legacy profile 未启用 updater | Bash、PowerShell 均不创建或启动 updater | +| 原 Watchtower 更新通道 | 重新开启轮询并发布 updater 新标签,Watchtower 仍实际更新 updater 1.1.0 → 1.2.0 | +| 业务服务对照 | 全程容器 ID 不变,没有被维护入口直接重建 | + +实际运行验收耗时 27.58 秒,不含镜像准备及修改前复现。证据:`after.json`、各场景 `.log`、`watchtower-final.log`。这是自建 registry、真实 Compose 和真实更新组件的运行结果。业务服务为独立 Node 对照容器,并未部署整套业务及数据库。 + +## 定向检查 + +- `python3 scripts/test-infrastructure-auto-update.py`:通过,19.496 秒。覆盖 Bash/PowerShell、Compose v2/v1 命令路径、单项失败不影响另一项、全部拉取失败不重建、legacy 跳过、并发排除、健康探测,以及原有迁移和 cron 去重保护。 +- `pwsh -NoProfile -File scripts/Test-InfrastructureUpdateTask.ps1`:通过,0.18 秒。验证注册参数及去重、关闭行为;计划任务 API 为测试替身。 +- Bash/PowerShell 语法、ShellCheck、diff 空白检查:通过,合计 0.72 秒。PowerShell UTF-8 BOM 保持不变,供 Windows PowerShell 5.1 使用。 + +证据:`validation.json`、`quality-checks.json` 及本轮工具输出。没有运行全量前端质量门禁,本轮未涉及前端业务代码。 + +## 验证边界 + +Windows 宿主机真实计划任务注册、生产六小时和 24 小时长周期、客户私有仓库网络及权限尚未验证。Compose v1 是命令适配行为验收,实际容器验收使用 Docker Compose v5.3.1。PowerShell 实际容器入口在 macOS 上运行。 + +历史终态任务持久化已实际验证;本轮没有在业务更新执行到一半时替换 updater 的故障演练,也不据此宣称所有活动任务恢复已验收。 + +Compose 根据启用的 profile 选择可维护服务,运行 `config --services` 已实测关闭 profile 时不包含 updater。避免直接显式启动未启用的 updater,因显式指定服务会自动启用其 profile。[Docker 官方说明](https://docs.docker.com/compose/how-tos/profiles/) + +本方案仍需 Docker 和镜像仓库可用;镜像启动失败不会自动回滚。每项维护有健康等待并报告失败,后续定时任务继续检查。 + +本次已清理四个自建验收容器及独立网络,原本机 `test-watchtower-1` 保持健康。证据:`cleanup.json`。 + +原始运行日志与入口快照保存在本机证据目录:`/Users/wangbo/.codex/visualizations/2026/10/07/01a11505-cae4-76e0-9ba9-235f6962b2b4/updater-double-insurance-20261008`。本目录附带修改前、修改后及定向检查的结构化结果。 diff --git a/system-update/releases/infrastructure-auto-update-20261008/after.json b/system-update/releases/infrastructure-auto-update-20261008/after.json new file mode 100644 index 0000000..f79be16 --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/after.json @@ -0,0 +1,53 @@ +{ + "passed": true, + "bashUpdatedUpdaterWithWatchtowerStoppedAndPullFailed": true, + "powershellUpdatedUpdaterWithWatchtowerStoppedAndPullFailed": true, + "failedServiceReportedWhileOtherServiceMaintained": true, + "persistedJobRetained": true, + "updaterPullFailurePreservedContainerAndWatchtowerRecovered": true, + "unchangedImagesNoRecreate": true, + "legacyProfileDoesNotStartUpdater": true, + "originalWatchtowerChannelStillUpdatesUpdater": true, + "businessContainerUnchanged": true, + "seconds": 27.58, + "before": [ + { + "name": "/easyai-updater-insurance-20261008-watchtower", + "id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c", + "image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7", + "running": false + }, + { + "name": "/easyai-updater-insurance-20261008-updater", + "id": "9837e6b0328798dbbad17ff310ef8d6d27761c1d152d374a91e56fdb3009c31e", + "image": "sha256:275e9ff7fb453d16feabfb9b92e1cd603f49c434bf2daa80a9f5ad6f505bffcb", + "running": true + }, + { + "name": "/easyai-updater-insurance-20261008-business", + "id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f", + "image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493", + "running": true + } + ], + "afterIndependentTimer": [ + { + "name": "/easyai-updater-insurance-20261008-watchtower", + "id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c", + "image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7", + "running": false + }, + { + "name": "/easyai-updater-insurance-20261008-updater", + "id": "f4b5029ae1bd5b9d33ef472039b3e938242ac68c5d3a36019035a745532a0734", + "image": "sha256:d7e4d04d4e614d537a08ee24f58317a3d6734e3514ca61eafc92bd1bb7a8d6ef", + "running": true + }, + { + "name": "/easyai-updater-insurance-20261008-business", + "id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f", + "image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493", + "running": true + } + ] +} \ No newline at end of file diff --git a/system-update/releases/infrastructure-auto-update-20261008/before.json b/system-update/releases/infrastructure-auto-update-20261008/before.json new file mode 100644 index 0000000..ae08f09 --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/before.json @@ -0,0 +1,28 @@ +{ + "reproduced": true, + "watchtowerStopped": true, + "watchtowerPullFailed": true, + "updaterRemainedOld": true, + "bashExitCode": 1, + "powershellExitCode": 1, + "containers": [ + { + "name": "/easyai-updater-insurance-20261008-watchtower", + "id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c", + "image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7", + "running": false + }, + { + "name": "/easyai-updater-insurance-20261008-updater", + "id": "9837e6b0328798dbbad17ff310ef8d6d27761c1d152d374a91e56fdb3009c31e", + "image": "sha256:275e9ff7fb453d16feabfb9b92e1cd603f49c434bf2daa80a9f5ad6f505bffcb", + "running": true + }, + { + "name": "/easyai-updater-insurance-20261008-business", + "id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f", + "image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493", + "running": true + } + ] +} \ No newline at end of file diff --git a/system-update/releases/infrastructure-auto-update-20261008/cleanup.json b/system-update/releases/infrastructure-auto-update-20261008/cleanup.json new file mode 100644 index 0000000..caa570d --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/cleanup.json @@ -0,0 +1,10 @@ +{ + "ownContainersRemoved": [ + "/easyai-updater-insurance-20261008-watchtower", + "/easyai-updater-insurance-20261008-updater", + "/easyai-updater-insurance-20261008-business", + "/easyai-updater-insurance-20261008-registry" + ], + "ownNetworkRemoved": true, + "originalLocalTestWatchtowerHealthy": true +} \ No newline at end of file diff --git a/system-update/releases/infrastructure-auto-update-20261008/quality-checks.json b/system-update/releases/infrastructure-auto-update-20261008/quality-checks.json new file mode 100644 index 0000000..051914e --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/quality-checks.json @@ -0,0 +1,7 @@ +{ + "BashSyntax": true, + "PowerShellSyntax": true, + "ShellCheck": true, + "DiffWhitespace": true, + "seconds": 0.72 +} \ No newline at end of file diff --git a/system-update/releases/infrastructure-auto-update-20261008/validation.json b/system-update/releases/infrastructure-auto-update-20261008/validation.json new file mode 100644 index 0000000..4393408 --- /dev/null +++ b/system-update/releases/infrastructure-auto-update-20261008/validation.json @@ -0,0 +1,14 @@ +{ + "infrastructureTests": { + "passed": true, + "seconds": 19.496 + }, + "powershellTaskContract": { + "passed": true, + "seconds": 0.18 + }, + "windowsPowerShellEncoding": { + "bomPreserved": true + }, + "fullQualityGateRun": false +} \ No newline at end of file diff --git a/system-update/releases/online-update-release-source-20261007/acceptance.md b/system-update/releases/online-update-release-source-20261007/acceptance.md new file mode 100644 index 0000000..560512f --- /dev/null +++ b/system-update/releases/online-update-release-source-20261007/acceptance.md @@ -0,0 +1,80 @@ +# 在线更新发布源修复与测试验收(2026-10-07) + +本次修复已在 `https://demo.51easyai.com` 完成真实浏览器验收。候选镜像和测试发布源只用于测试环境;验收时尚未提交代码;该测试验收未更新正式 `latest` 镜像和仓库根目录的正式发布清单。 + +## 已复现问题与根因 + +1. 测试站运行 `3.22.0.20260930.3`,页面从官方站运行版本接口检测到 `3.23.1.20261006`。 +2. 浏览器点击“立即更新”,任务 `bcaa8c58-3628-42fc-af02-bc5f97bbc1c2` 返回 `no_update`,实际版本和核心容器均未改变;页面仍提示有新版本。 +3. 更新器实际读取 `/var/lib/easyai-updater/release-manifest.json`,该路径由 `data/deployment-config/legacy-compose.json` 覆盖,内容仍指向旧版镜像。更新器不获取远程发布清单。 +4. 页面检测来源与执行来源不同;前端没有核对任务实际提供的版本,误把旧清单的“无需更新”作为新版已经完成更新。 +5. 正式部署仓库的发布清单也仍为 `3.22.0.20261002.2`;镜像仓库有后端 `3.23.1.20261006` 标签,但没有同版本前端标签。单纯改成本地另一条清单路径不能解决发布信息不一致。 + +## 最小修复及兼容性 + +- 更新器默认从部署仓库 `main` 分支获取正式清单;私有源使用 `UPDATE_MANIFEST_URL`,离线部署须显式选择 `UPDATE_MANIFEST_SOURCE=local`。默认远程模式忽略历史本地路径覆盖,获取失败不会退回旧清单。 +- 受控模式的版本检测通过更新器获取可安装版本,不再取官方站当前运行版本;旧 Watchtower 模式保留原协议和检测缓存。 +- 新版客户端提交检测到的版本与发布标识;执行前重新获取清单,目标发生变化时拒绝安装另一版本。旧客户端只传 `action` 的请求继续兼容。 +- 任务保存完整清单,恢复及回滚使用同一快照。前端核对成功或“无需更新”的实际版本,避免错误提示;普通更新、强制更新和管理员新版提醒成功后均刷新检测结果。 +- 只对更高版本显示新版提示;受控更新拒绝自动回退到更低版本,包括强制更新。发布源不可用时显示“暂时无法检查”和“重新检查”。 +- 主要调用方:系统设置更新卡片、管理员新版提醒、任务恢复入口、自动更新调度及旧手动接口;未增加业务 Provider 或改变自动更新开关。 + +责任代码与本次改动位置: + +- 后端 `apps/easyai-updater/src/release-source.ts`、`update-runner.ts`、`main.ts`:发布源、目标固定、快照恢复与内部发布信息接口。 +- 后端 `apps/easyai-server-main/src/app.service.ts`、`system-update/managed-update.service.ts`、`system-update/system-update.controller.ts`:统一检测来源、兼容请求及目标字段传递。 +- 前端 `composables/system-update/useSystemUpdateJob.ts`:核对更新结果与检测目标;`pages/admin/sys/operationConfig.vue` 和 `components/product-update/ProductUpdateAdminTouchpoint.vue`:版本比较、检测异常提示及成功后刷新。 +- 部署仓库 `docker-compose.yml`、`.env.sample`:默认远程发布源与显式离线选项;原正式清单保留不变。 + +## 真实运行验收 + +先部署保留旧版本号的修复版控制入口作为测试基线;前端基线使用独立镜像,使实际任务能够验证网页容器替换。最终安装的镜像来自本次工作区构建。候选清单见同目录 `release-manifest.candidate.json`,不作为正式发布清单。 + +| 场景 | 结果 | +| --- | --- | +| 管理员通过真实登录界面进入系统设置并点击立即更新 | 只提交一次请求;任务 `5a2968c5-9615-4e1f-9d88-79ada0f6aa06` 成功,页面约 88.9 秒完成反馈 | +| 版本与容器核对 | `3.22.0.20260930.3` → `3.24.0.20261007`;后端和网页容器实际替换,实时服务镜像无需变化 | +| 页面刷新 | 显示 `3.24.0.20261007`、已是最新,不重复弹出新版提醒 | +| 同版本普通更新 | 返回 `no_update`,不重复替换容器 | +| 强制更新确认后取消 | 不创建任务 | +| 发布源真实返回 HTTP 503 | 页面显示暂时无法检查,不能误报已是最新;恢复源后点击重新检查恢复正常 | +| 发布源改成旧正式清单 | 不提示有新版;点击强制更新后,任务 `92182a3e-1948-4d82-99bc-aacb7c1c601f` 在镜像拉取前被拒绝,原因 `release_older_than_current` | +| 配置保留 | 核心服务环境变量、完整容器配置(除镜像)、端口、挂载路径及权限、网络地址和别名保持一致;宿主机 `.env` 与 `docker-compose.yml` 哈希不变 | +| 自动更新设置 | 验收前后均为开启,未改变原开关 | +| 更新器实际重启 | 已完成任务完整保留,恢复后仍提供同一测试发布信息;四个相关容器均健康 | + +首次候选镜像补装运行依赖时移除了内部共享包,启动检查失败后恢复了原服务。补齐共享包并通过加载检查后,重新部署健康基线,再进行上述正式浏览器验收;没有把失败启动计为升级成功。 + +## 定向检查 + +本地检查使用 Node `22.22.0`、Yarn `1.22.22`,共 79 个不同的定向行为用例通过: + +| 检查 | 结果与耗时 | +| --- | --- | +| updater Node 测试 | 14 个通过,约 1.6 秒;覆盖真实 HTTP 发布源、源失败、目标变化、快照恢复及旧版回退保护 | +| 后端 system-update Jest | 38 个通过,约 2.3 秒;包含新旧检测协议、实际 DTO 校验、自动更新调度和旧手动接口 | +| 前端任务与实际挂载测试 | 首轮 13 个通过,约 49 秒;追加后的设置与更新处理函数 5 个通过,约 4 秒,其中新增强制更新用例 1 个 | +| 版本工具和产品版本测试 | 13 个通过,约 0.13 秒 | +| 部署脚本隔离 dry-run | 通过,约 4.3 秒,确认默认远程源及原部署契约 | +| 后端及更新器构建 | 通过,约 25 秒 | +| 最终前端生产构建 | 通过,约 168 秒 | + +隔离环境中的旧实现会失败:前端将旧版 `no_update` 误判成功;更新器六个新增发布源场景不能满足预期。相同失败用例在修复后通过。HTTP 发布源测试使用受控 Docker 替身,真实容器替换另外通过测试站验收;没有用源码断言或构建替代运行验收。 + +## 证据与测试环境维护 + +完整本地证据目录:`/Users/wangbo/.codex/visualizations/2026/10/07/01a11505-cae4-76e0-9ba9-235f6962b2b4/system-update-diagnosis`。 + +- `acceptance-summary.json`、`browser-upgrade-acceptance.json`、`browser-acceptance.json`:实际请求、任务、版本及异常场景。 +- `acceptance-update-success.png`、`acceptance-final-card.png`、`acceptance-feed-unavailable.png`、`acceptance-older-release-blocked.png`:真实页面结果。 +- `config-before-browser.json`、`config-after-browser.json`、`mount-verification.json`、`final-verification.json`:配置、健康及任务持久化核对。Docker 返回的挂载数组顺序不稳定,已按内容核对路径和权限一致。 +- `*-tests.log`、`*-build*.log`、`updater-before-tests.log`、`frontend-before-tests.log`:定向检查与修复前失败记录。 + +测试环境保留内部测试发布源容器 `easyai-update-release-feed`,无新增宿主机端口。任务专用配置位于 `/root/easyai/data/update-acceptance-20261007`,其中 `accepted.json` 和更新后的 `baseline.json` 指向实际验收镜像;`rollback.json` 保留原镜像引用。维护测试站时须使用原 Compose、原 legacy override 与该目录下 `accepted.json` 的组合,不能将该测试清单误发布成正式清单。 + +## 未验证范围及正式发布条件 + +- 未发布生产,未执行全站业务回归或全量质量门禁;健康检查和更新入口验收不能代表全部业务兼容性。 +- 定时自动更新实际到点执行未验收,调度逻辑已做定向测试;自动更新设置保持原值。 +- 已测试持久化快照恢复,并实际重启验证已完成任务;没有在真实容器替换过程中故意中断 updater。旧 updater 未带清单快照的在途任务跨版本迁移尚未验收,升级更新器前应先确认没有执行中的更新任务。 +- 正式发布时须一起交付 updater 1.2.0、后端和前端,并在镜像完整发布后同步正式清单的版本、发布标识、不可变摘要及大小。只改官方站版本或只推送 `latest` 仍无法代表客户可安装的新版本。 diff --git a/system-update/releases/online-update-release-source-20261007/release-manifest.candidate.json b/system-update/releases/online-update-release-source-20261007/release-manifest.candidate.json new file mode 100644 index 0000000..5f6df6b --- /dev/null +++ b/system-update/releases/online-update-release-source-20261007/release-manifest.candidate.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 1, + "channel": "stable", + "releaseId": "update-source-acceptance-20261007-r3", + "productVersion": "3.24.0.20261007", + "platform": "linux/amd64", + "publishedAt": "2026-10-07T06:45:45.630545+00:00", + "managedUpdateAllowed": true, + "scheduledUpdateAllowed": true, + "rollbackSafe": true, + "minimumUpdaterVersion": "1.2.0", + "minimumDeploymentSchemaVersion": 2, + "requiredFreeBytes": 8589934592, + "components": [ + { + "key": "server-main", + "image": "registry.cn-shanghai.aliyuncs.com/comfy-ai/comfy-server", + "digest": "sha256:b334362777cfc2a1064a264cf10dac2766b32674627d98775be8a2d9ceb7da5b", + "imageSizeBytes": 2747248955, + "buildId": "test-source-20261007" + }, + { + "key": "ws-gateway", + "image": "registry.cn-shanghai.aliyuncs.com/easyaigc/wsgateway", + "digest": "sha256:a2b5fb67384c7f0eca7c7a768b5deacb4c0d2195dfbeb6e16ef40d6dc097bf25", + "imageSizeBytes": 1982472178, + "buildId": "test-source-20261007" + }, + { + "key": "web", + "image": "registry.cn-shanghai.aliyuncs.com/comfy-ai/one-ai", + "digest": "sha256:9ea88a929e743dcba2c7a38402c913d011af99b3ccc63a9cf8b4f97e4abda4d5", + "imageSizeBytes": 886998757, + "buildId": "test-source-20261007" + } + ] +} \ No newline at end of file diff --git a/update.ps1 b/update.ps1 index ba17edf..cf84b8d 100644 --- a/update.ps1 +++ b/update.ps1 @@ -215,6 +215,7 @@ if (-not (Get-EnvValue $environmentContent "SYSTEM_UPDATE_MODE")) { & (Join-Path $scriptDir "scripts\Apply-UpdateEnvMigrations.ps1") ` -EnvPath (Join-Path $scriptDir ".env") ` -ProjectRoot $scriptDir +& (Join-Path $scriptDir "scripts/Initialize-InfrastructureUpdateEnv.ps1") -ProjectRoot $scriptDir . (Join-Path $scriptDir "scripts\Initialize-ServerHttpBindIp.ps1") Initialize-ServerHttpBindIp -Path (Join-Path $scriptDir ".env") @@ -390,6 +391,8 @@ if ($hasComposeV2) { Wait-CoreServices +& (Join-Path $scriptDir "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $scriptDir + Write-Host "" Write-Host "================================" Write-Host " 更新完成" diff --git a/update.sh b/update.sh index 00ade56..6b2ad14 100755 --- a/update.sh +++ b/update.sh @@ -180,6 +180,7 @@ init_server_http_bind_ip .env # shellcheck source=scripts/init-security-env.sh . ./scripts/init-security-env.sh init_security_env .env upgrade +bash ./scripts/init-infrastructure-update-env.sh "$SCRIPT_DIR" if [ -n "${UPDATE_RELEASE_VERSION:-}" ]; then write_env_value .env VERSION "$UPDATE_RELEASE_VERSION" @@ -337,4 +338,5 @@ fi if ! wait_for_core_services; then exit 1 fi +bash ./scripts/install-infrastructure-update-task.sh "$SCRIPT_DIR" echo "🎉 EasyAI 应用更新成功"