feat(deploy): 为更新组件增加定时维护与双保险
This commit is contained in:
+7
-1
@@ -261,6 +261,9 @@ COMPOSE_PROFILES=managed-update
|
||||
WATCHTOWER_CORE_UPDATE_ENABLED=false
|
||||
# 更新器由独立于任务执行进程的 Watchtower 每 24 小时检查一次，避免自替换中断任务。
|
||||
WATCHTOWER_UPDATE_TARGET=easyai-updater
|
||||
UPDATE_MANIFEST_SOURCE=remote
|
||||
# 留空时使用部署仓库中的官方稳定发布清单；离线部署可显式选择 local。
|
||||
UPDATE_MANIFEST_URL=
|
||||
SYSTEM_UPDATER_UPDATE_INTERVAL_SECONDS=86400
|
||||
WATCHTOWER_CLEANUP=true
|
||||
# 核心服务替换后最多等待 5 分钟恢复可用。
|
||||
@@ -268,7 +271,10 @@ UPDATE_HEALTH_TIMEOUT_MS=300000
|
||||
# 私有镜像仓库登录信息；执行 docker login 后通常位于此处。
|
||||
DOCKER_CONFIG_FILE=/root/.docker/config.json
|
||||
SYSTEM_UPDATE_INTERNAL_TOKEN=
|
||||
WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3
|
||||
# 宿主机每六小时分别维护 Watchtower 和 updater；保留 Watchtower 的 updater 更新计划。
|
||||
INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true
|
||||
WATCHTOWER_SCOPE=easyai
|
||||
WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest
|
||||
# 更新接口及事件接口仅供容器内网访问，由初始化脚本生成独立令牌。
|
||||
WATCHTOWER_API_KEY=
|
||||
WATCHTOWER_HTTP_API_EVENTS_TOKEN=
|
||||
|
||||
@@ -7,6 +7,10 @@ on:
|
||||
- 'update.sh'
|
||||
- 'docker-compose.yml'
|
||||
- 'easyai-proxy.conf.sample'
|
||||
- 'scripts/*infrastructure*.sh'
|
||||
- 'scripts/*Infrastructure*.ps1'
|
||||
- 'scripts/test-infrastructure-auto-update.py'
|
||||
- 'scripts/update-env.d/*.env'
|
||||
- '.env.sample'
|
||||
- 'scripts/init-public-api-base-url.sh'
|
||||
- 'scripts/test-public-api-base-url.sh'
|
||||
@@ -25,6 +29,10 @@ on:
|
||||
- 'update.sh'
|
||||
- 'docker-compose.yml'
|
||||
- 'easyai-proxy.conf.sample'
|
||||
- 'scripts/*infrastructure*.sh'
|
||||
- 'scripts/*Infrastructure*.ps1'
|
||||
- 'scripts/test-infrastructure-auto-update.py'
|
||||
- 'scripts/update-env.d/*.env'
|
||||
- '.env.sample'
|
||||
- 'scripts/init-public-api-base-url.sh'
|
||||
- 'scripts/test-public-api-base-url.sh'
|
||||
@@ -62,3 +70,9 @@ jobs:
|
||||
|
||||
- name: Test deployment dry-run matrix
|
||||
run: bash scripts/test-deployment-public-url.sh
|
||||
|
||||
- name: Test automatic Watchtower maintenance
|
||||
run: |
|
||||
python3 scripts/test-infrastructure-auto-update.py
|
||||
pwsh -NoProfile -File scripts/Test-InfrastructureUpdateTask.ps1
|
||||
bash scripts/test-update-env-migrations.sh
|
||||
|
||||
@@ -14,6 +14,10 @@ on:
|
||||
- 'scripts/Test-SecurityOrigin.ps1'
|
||||
- 'scripts/Assert-RedisPersistence.ps1'
|
||||
- 'scripts/Test-RedisPersistence.ps1'
|
||||
- 'scripts/*infrastructure*.sh'
|
||||
- 'scripts/*Infrastructure*.ps1'
|
||||
- 'scripts/test-infrastructure-auto-update.py'
|
||||
- 'scripts/update-env.d/*.env'
|
||||
- '.env.sample'
|
||||
- '.github/workflows/test-start-ps1.yml'
|
||||
pull_request:
|
||||
@@ -28,6 +32,10 @@ on:
|
||||
- 'scripts/Test-SecurityOrigin.ps1'
|
||||
- 'scripts/Assert-RedisPersistence.ps1'
|
||||
- 'scripts/Test-RedisPersistence.ps1'
|
||||
- 'scripts/*infrastructure*.sh'
|
||||
- 'scripts/*Infrastructure*.ps1'
|
||||
- 'scripts/test-infrastructure-auto-update.py'
|
||||
- 'scripts/update-env.d/*.env'
|
||||
- '.env.sample'
|
||||
- '.github/workflows/test-start-ps1.yml'
|
||||
workflow_dispatch:
|
||||
@@ -84,3 +92,6 @@ jobs:
|
||||
run: |
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\Test-SecurityOrigin.ps1
|
||||
powershell -ExecutionPolicy Bypass -File .\scripts\Test-RedisPersistence.ps1
|
||||
|
||||
- name: Test Watchtower schedule registration contract
|
||||
run: powershell -ExecutionPolicy Bypass -File .\scripts\Test-InfrastructureUpdateTask.ps1
|
||||
|
||||
@@ -184,9 +184,11 @@ update.sh 脚本用于自动更新 EasyAI 应用，包含以下功能：
|
||||
|
||||
新安装默认启用可查询进度的在线更新模式。管理员在页面发起更新后，即使主服务重启导致原请求断开，页面也会根据任务编号继续查询最终结果。存量部署如果没有 `easyai-updater` 容器，更新脚本会保留 Watchtower 模式；已经显式设置 `SYSTEM_UPDATE_MODE=legacy_watchtower` 的环境也保持该模式。只有新安装默认启用 updater。新模式只管理 `easyai-server`、`easyai-web` 和 `ws-gateway`，数据库、Redis、RabbitMQ 等基础服务仍按原部署流程维护。
|
||||
|
||||
更新执行器 `easyai-updater` 由 Watchtower 单独守护，默认每 24 小时检查并更新自身。它的任务状态保存在宿主机 `data/update-state` 中，自身更新或意外重启后会继续恢复未完成任务；Watchtower 不再接管三个核心服务，因此页面任务结果不会与后台静默替换相互冲突。
|
||||
更新执行器 `easyai-updater` 由 Watchtower 默认每 24 小时检查更新，同时由宿主机定时任务每六小时直接检查更新，提供两条独立维护路径。它的任务状态保存在宿主机 `data/update-state` 中，自身更新或意外重启后会继续恢复未完成任务；Watchtower 不再接管三个核心服务，因此页面任务结果不会与后台静默替换相互冲突。
|
||||
|
||||
受控更新使用仓库内的 `system-update/release-manifest.json` 作为稳定发布清单，Compose 会以只读方式挂载给 updater。发布新正式镜像时必须同步更新清单中的不可变摘要、镜像大小和版本标识；服务器执行 `update.sh` 拉取部署仓库后即可获得清单，不需要手工向运行目录复制文件。
|
||||
受控更新默认直接获取部署仓库 `main` 分支的 `system-update/release-manifest.json`，无需先在服务器执行 `git pull`。页面检测、手动更新和自动更新使用同一发布清单；任务开始后将清单保存在任务记录中，服务重启恢复时继续使用原目标。发布新正式镜像时必须同步发布清单中的不可变摘要、镜像大小和版本标识，不能只修改官方站运行版本或推送 `latest` 镜像。
|
||||
|
||||
发布源不可用时会提示暂时无法检查更新，并保留当前服务；不会用历史清单宣称已是最新。`UPDATE_MANIFEST_URL` 可配置私有发布源，留空时使用官方部署仓库。离线部署可显式设置 `UPDATE_MANIFEST_SOURCE=local`，此时读取 Compose 只读挂载的本地清单；需要自行维护该清单。旧部署中的 `UPDATE_MANIFEST_FILE` 路径覆盖不再影响默认远程模式。
|
||||
|
||||
新安装或已有 updater 的部署无需手工编辑 `.env`。更新脚本会从 `scripts/update-env.d/010-managed-system-update.env` 应用以下缺省值：
|
||||
|
||||
@@ -203,7 +205,11 @@ EASYAI_DEPLOYMENT_SCHEMA_VERSION=2
|
||||
|
||||
`SYSTEM_UPDATE_INTERNAL_TOKEN` 由 `start.sh` / `update.sh` 自动生成并持久化，不需要手工填写。`UPDATE_HEALTH_TIMEOUT_MS` 控制每个核心服务替换后的恢复等待时间，默认 300000 毫秒。私有镜像仓库需要先执行 `docker login`；更新器与仅负责其自更新的 Watchtower 都会只读挂载 `DOCKER_CONFIG_FILE` 指向的 Docker 登录配置。切换后执行一次 `./update.sh`，确认 `easyai-updater` 以及三个核心服务均处于可用状态。
|
||||
|
||||
Watchtower 固定使用 `1.22.3`。内网启用更新、事件、历史、状态和健康接口，不映射主机端口。`WATCHTOWER_API_KEY` 与 `WATCHTOWER_HTTP_API_EVENTS_TOKEN` 是两个独立随机令牌，由 Bash 和 PowerShell 安全初始化流程补齐；浏览器只访问后端，不接触这些令牌。managed 模式只扫描 updater 并定期检查自身；Watchtower 模式关闭定期扫描，只在用户或系统明确提交更新时更新核心服务，避免任务与定时扫描无法对应。
|
||||
Watchtower 默认使用 `latest`。`start.sh/update.sh` 自动安装宿主机 cron，`start.ps1/update.ps1` 自动注册 Windows 计划任务；每六小时分别对 Watchtower 和已启用的 updater 执行 `docker compose pull <服务>`、`docker compose up -d --no-deps <服务>` 并确认健康状态。只在镜像或配置有变化时重建，拉取失败不会停止该服务的现有容器。一项维护失败仍继续处理另一项，日志保留各自结果并报告失败；因此 updater 的宿主机维护不依赖 Watchtower 可用。未启用 updater 的 legacy 部署跳过 updater，不自动切换更新模式。Watchtower 的 updater 24 小时更新计划保持不变，无需额外容器。Windows 任务使用当前登录用户，以便访问其 Docker Desktop；Docker 不可用时保留失败日志，下次计划重试。
|
||||
|
||||
内网启用更新、事件、历史、状态和健康接口，不映射主机端口。`WATCHTOWER_API_KEY` 与 `WATCHTOWER_HTTP_API_EVENTS_TOKEN` 是两个独立随机令牌，由 Bash 和 PowerShell 安全初始化流程补齐；浏览器只访问后端。新安装及默认整组升级使用独立 `WATCHTOWER_SCOPE`（`easyai`），防止启动时清理其他部署的 Watchtower；定向升级和自定义监控目标保留原 scope，避免未重建的容器标签与新范围不匹配。managed 模式只扫描 updater，legacy 模式仍只在明确提交业务更新时执行原更新链路。
|
||||
|
||||
定时任务按部署目录去重，重复运行部署脚本不会重复注册。最近一次结果写入 `data/infrastructure-update/last-run.log`。设置 `INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false` 后再次执行部署脚本可移除本部署的定时任务；不改变 updater 的原有维护计划。启用时，官方镜像的旧版本标签自动迁移到 `latest`，自定义仓库或镜像摘要保持原值。旧客户需先由运维执行一次新版部署脚本，之后不用客户手工升级 Watchtower；只发布仓库不会自动把宿主机定时任务安装到未接入的机器。发布到官方 Watchtower `latest` 的镜像应先通过部署及旧接口兼容验收。
|
||||
|
||||
两个页面入口共用计时和步骤窗口。updater 保存阶段完成时间到 `data/update-state/jobs`；无 updater 时后端保存任务和单任务锁到 MongoDB。刷新或重新登录后按站点与账号恢复任务查询。请求超时或服务断开不会自动创建另一项更新，旧接口无法读取的阶段会明确提示。后端重启时页面保留进度，提醒在 3–5 分钟后刷新检查版本；只有确认目标版本和服务恢复才显示完成。
|
||||
|
||||
|
||||
+14
-1
@@ -9,6 +9,7 @@ services:
|
||||
#端口冲突时，只需要修改前面的端口比如修改为，3011:3010
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
ports:
|
||||
- "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${WEB_PORT}:3010"
|
||||
# read_only: true
|
||||
@@ -60,6 +61,7 @@ services:
|
||||
image: registry.cn-shanghai.aliyuncs.com/comfy-ai/comfy-server:${VERSION} # 阿里云
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
ports:
|
||||
- "${SERVER_HTTP_BIND_IP:-127.0.0.1}:${SERVER_HTTP_PORT}:3001" # Linux Nginx 部署默认仅回环；Windows/LAN 直连由启动脚本显式覆盖
|
||||
# read_only: true
|
||||
@@ -166,6 +168,7 @@ services:
|
||||
image: registry.cn-shanghai.aliyuncs.com/easyaigc/model-conversion-worker:${MODEL_CONVERSION_WORKER_VERSION:-latest}
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
restart: unless-stopped
|
||||
cpus: 2
|
||||
mem_limit: 4g
|
||||
@@ -219,6 +222,7 @@ services:
|
||||
image: registry.cn-shanghai.aliyuncs.com/easyaigc/wsgateway:latest # 阿里云
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=${WATCHTOWER_CORE_UPDATE_ENABLED:-true}"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
ports:
|
||||
- "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${CONFIG_WS_PORT:-3002}:3002" # Linux IP/域名模式默认仅供同机 Nginx 访问
|
||||
read_only: true
|
||||
@@ -335,6 +339,7 @@ services:
|
||||
image: ${RABBITMQ_IMAGE:-registry.cn-shanghai.aliyuncs.com/easyaigc/mq:latest}
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
container_name: rabbitmq
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
@@ -349,7 +354,7 @@ services:
|
||||
easyai:
|
||||
ipv4_address: 172.21.0.12
|
||||
watchtower:
|
||||
image: ${WATCHTOWER_IMAGE:-registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3}
|
||||
image: ${WATCHTOWER_IMAGE:-registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest}
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ${DOCKER_CONFIG_FILE:-/root/.docker/config.json}:/config.json:ro
|
||||
@@ -358,6 +363,7 @@ services:
|
||||
environment:
|
||||
- WATCHTOWER_POLL_INTERVAL=${SYSTEM_UPDATER_UPDATE_INTERVAL_SECONDS:-86400}
|
||||
- WATCHTOWER_CLEANUP=${WATCHTOWER_CLEANUP:-false}
|
||||
- WATCHTOWER_SCOPE=${WATCHTOWER_SCOPE:-}
|
||||
- WATCHTOWER_HTTP_API_HOST=0.0.0.0
|
||||
- WATCHTOWER_HTTP_API_ENDPOINTS=health,update,metrics,events,history
|
||||
- WATCHTOWER_HTTP_API_TOKEN=${WATCHTOWER_API_KEY}
|
||||
@@ -366,6 +372,7 @@ services:
|
||||
restart: always
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=false"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
# ports:
|
||||
# - "${WATCHTOWER_PORT}:8080"
|
||||
read_only: true
|
||||
@@ -385,6 +392,7 @@ services:
|
||||
- managed-update
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
cap_drop:
|
||||
@@ -399,6 +407,8 @@ services:
|
||||
- DOCKER_CONFIG_FILE=/run/secrets/docker-config.json
|
||||
- UPDATE_STATE_DIR=/var/lib/easyai-updater
|
||||
- UPDATE_MANIFEST_FILE=/etc/easyai-updater/release-manifest.json
|
||||
- UPDATE_MANIFEST_SOURCE=${UPDATE_MANIFEST_SOURCE:-remote}
|
||||
- UPDATE_MANIFEST_URL=${UPDATE_MANIFEST_URL:-}
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ${DOCKER_CONFIG_FILE:-/root/.docker/config.json}:/run/secrets/docker-config.json:ro
|
||||
@@ -482,6 +492,7 @@ services:
|
||||
# 不映射宿主机端口，仅允许 Docker 内网中的 easyai-server 访问
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
volumes:
|
||||
- ./sandbox-data:/app/data
|
||||
- ./local_packages:/app/local_packages
|
||||
@@ -554,6 +565,7 @@ services:
|
||||
image: registry.cn-shanghai.aliyuncs.com/easyaigc/agent-governance:${ASG_VERSION:-latest}
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
ports:
|
||||
- "${EASYAI_PROXY_BIND_IP:-127.0.0.1}:${ASG_PORT:-3003}:3003"
|
||||
networks:
|
||||
@@ -585,6 +597,7 @@ services:
|
||||
image: registry.cn-shanghai.aliyuncs.com/easyaigc/agent-memory:${AMS_VERSION:-latest}
|
||||
labels:
|
||||
- "com.centurylinklabs.watchtower.enable=true"
|
||||
- "com.centurylinklabs.watchtower.scope=${WATCHTOWER_SCOPE:-}"
|
||||
ports:
|
||||
- "${EASYAI_INFRA_BIND_IP:-127.0.0.1}:${AMS_PORT:-3004}:3004"
|
||||
environment:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
param([Parameter(Mandatory=$true)][string]$ProjectRoot)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
|
||||
$path = Join-Path $ProjectRoot '.env'
|
||||
$content = [System.IO.File]::ReadAllText($path)
|
||||
if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return }
|
||||
$image = Get-EnvValue $content 'WATCHTOWER_IMAGE'
|
||||
if ([string]::IsNullOrWhiteSpace($image) -or $image -match '^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:\d+\.\d+\.\d+$') {
|
||||
$content = Set-EnvValue $content 'WATCHTOWER_IMAGE' 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest'
|
||||
[System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
|
||||
# Preserve old scope during a targeted update; unchanged containers keep their labels.
|
||||
$image = Get-EnvValue $content 'WATCHTOWER_IMAGE'
|
||||
$target = Get-EnvValue $content 'WATCHTOWER_UPDATE_TARGET'
|
||||
$scope = Get-EnvValue $content 'WATCHTOWER_SCOPE'
|
||||
if ([string]::IsNullOrWhiteSpace($env:UPDATE_SERVICES) -and [string]::IsNullOrWhiteSpace($scope) -and $image -eq 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' -and $target -in @('', 'easyai-updater', 'easyai-server easyai-wsgateway easyai-web')) {
|
||||
$content = Set-EnvValue $content 'WATCHTOWER_SCOPE' 'easyai'
|
||||
[System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
param(
|
||||
[Parameter(Mandatory=$true)][string]$ProjectRoot,
|
||||
[switch]$DryRun
|
||||
)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProjectRoot = (Resolve-Path $ProjectRoot).Path
|
||||
. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
|
||||
$enabled = (Get-EnvValue ([System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env'))) 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -ne 'false'
|
||||
$hash = [System.Security.Cryptography.SHA256]::Create()
|
||||
try { $id = ([System.BitConverter]::ToString($hash.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($ProjectRoot)))).Replace('-', '').Substring(0, 12) }
|
||||
finally { $hash.Dispose() }
|
||||
$name = "EasyAI-Watchtower-$id"
|
||||
if ($DryRun -and -not $enabled) { Write-Host "更新组件定时任务（dry-run）：已关闭"; return }
|
||||
if ($DryRun) { Write-Host "Watchtower 和 updater 定时任务（dry-run）：$name，每六小时检查一次"; return }
|
||||
if (-not $enabled) {
|
||||
$existing = Get-ScheduledTask -TaskName $name -ErrorAction SilentlyContinue
|
||||
if ($existing) { Unregister-ScheduledTask -TaskName $name -Confirm:$false }
|
||||
return
|
||||
}
|
||||
$executable = (Get-Process -Id $PID).Path
|
||||
$script = Join-Path $ProjectRoot 'scripts/Update-Infrastructure.ps1'
|
||||
$action = New-ScheduledTaskAction -Execute $executable -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$script`" -ProjectRoot `"$ProjectRoot`""
|
||||
$triggers = @('00:17', '06:17', '12:17', '18:17') | ForEach-Object { New-ScheduledTaskTrigger -Daily -At $_ }
|
||||
# Docker Desktop belongs to the logged-in user, not the SYSTEM account.
|
||||
$user = [System.Environment]::UserName
|
||||
if ($env:USERDOMAIN) { $user = "$env:USERDOMAIN\$user" }
|
||||
$principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest
|
||||
$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 2)
|
||||
Register-ScheduledTask -TaskName $name -Action $action -Trigger $triggers -Principal $principal -Settings $settings -Force | Out-Null
|
||||
Write-Host '已配置 Watchtower 和 updater 定时任务：每六小时检查一次（Docker Desktop 可用时执行）'
|
||||
@@ -0,0 +1,31 @@
|
||||
# Test scheduler registration without changing the host's real scheduled tasks.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$started = Get-Date
|
||||
$root = Join-Path ([System.IO.Path]::GetTempPath()) ('easyai-task-' + [guid]::NewGuid().ToString('N'))
|
||||
[System.IO.Directory]::CreateDirectory((Join-Path $root 'scripts')) | Out-Null
|
||||
try {
|
||||
Copy-Item (Join-Path $PSScriptRoot 'Initialize-SecurityEnv.ps1') (Join-Path $root 'scripts')
|
||||
Copy-Item (Join-Path $PSScriptRoot 'Install-InfrastructureUpdateTask.ps1') (Join-Path $root 'scripts')
|
||||
$global:infraTaskTestTasks = @{ 'Unrelated-Backup' = @{ keep = $true } }
|
||||
function New-ScheduledTaskAction { param($Execute, $Argument); return @{Execute=$Execute; Argument=$Argument} }
|
||||
function New-ScheduledTaskTrigger { param([switch]$Daily, $At); return @{Daily=$Daily; At=$At} }
|
||||
function New-ScheduledTaskPrincipal { param($UserId, $LogonType, $RunLevel); return @{User=$UserId; Logon=$LogonType; Level=$RunLevel} }
|
||||
function New-ScheduledTaskSettingsSet { param([switch]$StartWhenAvailable, $MultipleInstances, $ExecutionTimeLimit); return @{Concurrent=$MultipleInstances; Available=$StartWhenAvailable} }
|
||||
function Register-ScheduledTask { param($TaskName, $Action, $Trigger, $Principal, $Settings, [switch]$Force); $global:infraTaskTestTasks[$TaskName]=@{Action=$Action; Trigger=$Trigger; Principal=$Principal; Settings=$Settings}; return @{} }
|
||||
function Get-ScheduledTask { param($TaskName, $ErrorAction); return $global:infraTaskTestTasks[$TaskName] }
|
||||
function Unregister-ScheduledTask { param($TaskName, $Confirm); $global:infraTaskTestTasks.Remove($TaskName) }
|
||||
$envFile = Join-Path $root '.env'
|
||||
[System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true')
|
||||
& (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
|
||||
& (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
|
||||
if ($global:infraTaskTestTasks.Count -ne 2) { throw 'Repeated install duplicated tasks or deleted unrelated tasks' }
|
||||
$key = @($global:infraTaskTestTasks.Keys | Where-Object { $_ -like 'EasyAI-Watchtower-*' })[0]
|
||||
$task = $global:infraTaskTestTasks[$key]
|
||||
if ($task.Trigger.Count -ne 4 -or $task.Settings.Concurrent -ne 'IgnoreNew') { throw 'Schedule or concurrency policy changed' }
|
||||
if ($task.Principal.Logon -ne 'Interactive') { throw 'Task would not use the Docker Desktop user' }
|
||||
if ($task.Action.Argument -notlike '*Update-Infrastructure.ps1*' -or $task.Action.Argument -notlike '*-NonInteractive*') { throw 'Incorrect task command' }
|
||||
[System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false')
|
||||
& (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
|
||||
if ($global:infraTaskTestTasks.Count -ne 1 -or -not $global:infraTaskTestTasks.ContainsKey('Unrelated-Backup')) { throw 'Disable removed an unrelated task' }
|
||||
Write-Host "PowerShell scheduler contract passed in $([math]::Round(((Get-Date)-$started).TotalSeconds,2)) seconds"
|
||||
} finally { Remove-Item -Recurse -Force $root; Remove-Variable infraTaskTestTasks -Scope Global -ErrorAction SilentlyContinue }
|
||||
@@ -0,0 +1,74 @@
|
||||
param([string]$ProjectRoot = (Split-Path $PSScriptRoot -Parent))
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-Location $ProjectRoot
|
||||
. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
|
||||
$content = [System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env'))
|
||||
if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return }
|
||||
$state = Join-Path $ProjectRoot 'data/infrastructure-update'
|
||||
[System.IO.Directory]::CreateDirectory($state) | Out-Null
|
||||
try { $lock = [System.IO.File]::Open((Join-Path $state 'task.lock'), 'OpenOrCreate', 'ReadWrite', 'None') }
|
||||
catch [System.IO.IOException] { return }
|
||||
$log = Join-Path $state 'last-run.log'
|
||||
$previousConfig = $env:DOCKER_CONFIG
|
||||
try {
|
||||
[System.IO.File]::WriteAllText($log, "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') 检查 Watchtower 和 updater 更新`n")
|
||||
$configFile = Get-EnvValue $content 'DOCKER_CONFIG_FILE'
|
||||
if (-not [string]::IsNullOrWhiteSpace($configFile) -and (Test-Path $configFile -PathType Leaf)) {
|
||||
$env:DOCKER_CONFIG = Split-Path $configFile -Parent
|
||||
}
|
||||
& docker compose version *> $null
|
||||
$v2 = $LASTEXITCODE -eq 0
|
||||
function Invoke-InfrastructureCompose {
|
||||
param([string[]]$ComposeArguments)
|
||||
if ($v2) { $output = & docker compose @ComposeArguments 2>&1 }
|
||||
else { $output = & docker-compose @ComposeArguments 2>&1 }
|
||||
$code = $LASTEXITCODE
|
||||
$output | Out-File $log -Append -Encoding utf8
|
||||
if ($code -ne 0) { throw '组件更新命令未成功，现有容器状态需查看日志' }
|
||||
}
|
||||
if ($v2) { $available = @(& docker compose config --services) }
|
||||
else { $available = @(& docker-compose config --services) }
|
||||
if ($LASTEXITCODE -ne 0 -or $available -notcontains 'watchtower') { throw '无法读取 Watchtower 部署配置' }
|
||||
$failed = @()
|
||||
foreach ($service in @('watchtower', 'easyai-updater')) {
|
||||
# Do not enable updater in profile-disabled legacy deployments.
|
||||
if ($available -notcontains $service) {
|
||||
"$service 未启用，跳过维护" | Out-File $log -Append -Encoding utf8
|
||||
continue
|
||||
}
|
||||
try {
|
||||
"检查 $service" | Out-File $log -Append -Encoding utf8
|
||||
Invoke-InfrastructureCompose -ComposeArguments @('pull', $service)
|
||||
Invoke-InfrastructureCompose -ComposeArguments @('up', '-d', '--no-deps', $service)
|
||||
$healthy = $false
|
||||
for ($attempt=0; $attempt -lt 12; $attempt++) {
|
||||
if ($v2) { $containerId = & docker compose ps -q $service }
|
||||
else { $containerId = & docker-compose ps -q $service }
|
||||
if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" }
|
||||
if ($containerId) {
|
||||
$status = & docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $containerId
|
||||
if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" }
|
||||
if ($status -eq 'true healthy') { $healthy = $true; break }
|
||||
if ($status -eq 'true none') {
|
||||
if ($service -eq 'watchtower') {
|
||||
& docker exec $containerId /watchtower --health-check *> $null
|
||||
} else {
|
||||
& docker exec $containerId node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))" *> $null
|
||||
}
|
||||
if ($LASTEXITCODE -eq 0) { $healthy = $true; break }
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
if (-not $healthy) { throw "$service 暂未恢复可用" }
|
||||
"$service 已恢复可用" | Out-File $log -Append -Encoding utf8
|
||||
} catch {
|
||||
"$service 本次维护未成功，下次定时任务将继续检查：$($_.Exception.Message)" | Out-File $log -Append -Encoding utf8
|
||||
$failed += $service
|
||||
}
|
||||
}
|
||||
if ($failed.Count -gt 0) { throw "更新组件维护未全部成功：$($failed -join ', ')" }
|
||||
} finally {
|
||||
$env:DOCKER_CONFIG = $previousConfig
|
||||
$lock.Dispose()
|
||||
}
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
. "$PROJECT_ROOT/scripts/init-security-env.sh"
|
||||
[ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
|
||||
image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)"
|
||||
if [ -z "$image" ] || [[ "$image" =~ ^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest
|
||||
fi
|
||||
# Scope changes require the selected containers to receive matching labels.
|
||||
# A targeted update must not strand unchanged containers outside the new scope.
|
||||
image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)"
|
||||
target="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_UPDATE_TARGET)"
|
||||
if [ -z "${UPDATE_SERVICES:-}" ] && [ -z "$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE)" ] && [ "$image" = registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest ]; then
|
||||
case "$target" in
|
||||
''|'easyai-updater'|'easyai-server easyai-wsgateway easyai-web')
|
||||
write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE easyai
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)"
|
||||
. "$PROJECT_ROOT/scripts/init-security-env.sh"
|
||||
# A path is shell-quoted and percent-escaped before embedding it in crontab.
|
||||
[[ "$PROJECT_ROOT" != *$'\n'* ]] || { echo '部署目录不能包含换行'; exit 1; }
|
||||
task_id="$(printf '%s' "$PROJECT_ROOT" | cksum | awk '{print $1}')"
|
||||
marker="# easyai-infrastructure-update-$task_id"
|
||||
job="$(printf '%q' "$PROJECT_ROOT/scripts/update-infrastructure.sh")"
|
||||
job="${job//%/\\%}"
|
||||
entry="17 */6 * * * PATH=/usr/local/bin:/opt/homebrew/bin:/usr/sbin:/usr/bin:/sbin:/bin /bin/bash $job >/dev/null 2>&1 $marker"
|
||||
if [ "${2:-}" = --dry-run ]; then
|
||||
if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then echo "更新组件定时任务（dry-run）：已关闭"; exit 0; fi
|
||||
printf 'Watchtower 和 updater 定时任务（dry-run）：%s\n' "$entry"
|
||||
exit 0
|
||||
fi
|
||||
cron() {
|
||||
if [ "$(id -u)" -eq 0 ]; then crontab "$@"; else sudo crontab "$@"; fi
|
||||
}
|
||||
if ! command -v crontab >/dev/null; then
|
||||
[ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
|
||||
if command -v apt-get >/dev/null; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y cron
|
||||
else
|
||||
echo '缺少 crontab，无法安装更新组件定时任务' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ "$(uname -s)" = Linux ]; then
|
||||
if [ -d /run/systemd/system ]; then
|
||||
sudo systemctl enable --now cron
|
||||
elif command -v service >/dev/null; then
|
||||
sudo service cron start
|
||||
fi
|
||||
fi
|
||||
error_file="$(mktemp)"
|
||||
trap 'rm -f "$error_file"' EXIT
|
||||
if ! current="$(LC_ALL=C cron -l 2>"$error_file")"; then
|
||||
if ! grep -qi 'no crontab for' "$error_file"; then
|
||||
echo '无法读取现有定时任务，未修改任务配置' >&2
|
||||
exit 1
|
||||
fi
|
||||
current=""
|
||||
fi
|
||||
# Keep all unrelated tasks and replace this deployment's single entry.
|
||||
next="$(printf '%s\n' "$current" | awk -v marker="$marker" 'substr($0, length($0)-length(marker)+1) != marker')"
|
||||
if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ]; then
|
||||
next="${next:+$next$'\n'}$entry"
|
||||
fi
|
||||
printf '%s\n' "$next" | cron -
|
||||
if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then
|
||||
echo '已移除本部署的更新组件定时任务'
|
||||
else
|
||||
echo '已配置 Watchtower 和 updater 定时任务：每六小时检查一次'
|
||||
fi
|
||||
@@ -20,6 +20,9 @@ mkdir -p "$TMP_DIR/scripts"
|
||||
mkdir -p "$TMP_DIR/system-update"
|
||||
cp \
|
||||
"$REPO_ROOT/scripts/init-security-env.sh" \
|
||||
"$REPO_ROOT/scripts/init-infrastructure-update-env.sh" \
|
||||
"$REPO_ROOT/scripts/install-infrastructure-update-task.sh" \
|
||||
"$REPO_ROOT/scripts/update-infrastructure.sh" \
|
||||
"$REPO_ROOT/scripts/init-security-origin.sh" \
|
||||
"$REPO_ROOT/scripts/init-public-api-base-url.sh" \
|
||||
"$REPO_ROOT/scripts/init-server-http-bind-ip.sh" \
|
||||
@@ -186,6 +189,8 @@ assert str(server_env.get("EASYAI_DEPLOYMENT_SCHEMA_VERSION")) == "2"
|
||||
assert str(updater_env.get("UPDATE_HEALTH_TIMEOUT_MS")) == "300000"
|
||||
assert updater_env.get("DOCKER_CONFIG_FILE") == "/run/secrets/docker-config.json"
|
||||
assert updater_env.get("UPDATE_MANIFEST_FILE") == "/etc/easyai-updater/release-manifest.json"
|
||||
assert updater_env.get("UPDATE_MANIFEST_SOURCE") == "remote"
|
||||
assert updater_env.get("UPDATE_MANIFEST_URL") == ""
|
||||
assert updater.get("labels", {}).get("com.centurylinklabs.watchtower.enable") == "true"
|
||||
assert watchtower.get("environment", {}).get("WATCHTOWER_POLL_INTERVAL") == "86400"
|
||||
assert watchtower.get("environment", {}).get("WATCHTOWER_CLEANUP") == "true"
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise real entry scripts in temporary projects; never edits host schedulers."""
|
||||
from pathlib import Path
|
||||
import json,os,subprocess,tempfile,time,shutil
|
||||
REPO=Path(__file__).resolve().parents[1]
|
||||
started=time.monotonic()
|
||||
with tempfile.TemporaryDirectory(prefix='easyai infrastructure % ') as tmp:
|
||||
root=Path(tmp); project=root/'project space';(project/'scripts').mkdir(parents=True)
|
||||
for name in ['init-security-env.sh','Initialize-SecurityEnv.ps1','init-infrastructure-update-env.sh','Initialize-InfrastructureUpdateEnv.ps1','install-infrastructure-update-task.sh','Install-InfrastructureUpdateTask.ps1','update-infrastructure.sh','Update-Infrastructure.ps1']:
|
||||
shutil.copy2(REPO/'scripts'/name,project/'scripts'/name)
|
||||
bins=root/'bin';bins.mkdir();calls=root/'calls.jsonl';cron=root/'crontab';cron.write_text('5 2 * * * existing-backup\n# existing comment\n')
|
||||
environment={**os.environ,'PATH':str(bins)+os.pathsep+os.environ['PATH'],'INFRA_TEST_CALLS':str(calls),'INFRA_TEST_CRON':str(cron)}
|
||||
def stub(name,body):
|
||||
p=bins/name;p.write_text(body);p.chmod(0o755)
|
||||
stub('sudo','#!/bin/sh\nexec "$@"\n')
|
||||
stub('service','#!/bin/sh\nexit 0\n')
|
||||
stub('systemctl','#!/bin/sh\nexit 0\n')
|
||||
stub('crontab','''#!/usr/bin/env python3
|
||||
import sys,os
|
||||
from pathlib import Path
|
||||
p=Path(os.environ['INFRA_TEST_CRON'])
|
||||
if os.environ.get('INFRA_TEST_CRON_ERROR'):
|
||||
print('permission denied',file=sys.stderr);sys.exit(1)
|
||||
if sys.argv[1]=='-l':print(p.read_text(),end='')
|
||||
else:p.write_text(sys.stdin.read())
|
||||
''')
|
||||
stub('docker','''#!/usr/bin/env python3
|
||||
import sys,os,json,time
|
||||
from pathlib import Path
|
||||
args=sys.argv[1:]
|
||||
with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(args)+'\\n')
|
||||
if args[:1]==['--config']:args=args[2:]
|
||||
if args==['compose','version']:sys.exit(1 if os.environ.get('INFRA_TEST_V1') else 0)
|
||||
if args[:3]==['compose','ps','-q']:print(args[3]+'-test-id')
|
||||
if args[:1]==['inspect']:print('true none' if os.environ.get('INFRA_TEST_NO_HEALTH') else 'true healthy')
|
||||
if args==['compose','config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
|
||||
if args[:2]==['compose','pull']:
|
||||
time.sleep(float(os.environ.get('INFRA_TEST_SLEEP','0')))
|
||||
if os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',args[2]]:sys.exit(1)
|
||||
''')
|
||||
stub('docker-compose','''#!/usr/bin/env python3
|
||||
import sys,os,json
|
||||
from pathlib import Path
|
||||
with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(['v1']+sys.argv[1:])+'\\n')
|
||||
if sys.argv[1:3]==['ps','-q']:print(sys.argv[3]+'-test-id')
|
||||
if sys.argv[1:]==['config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
|
||||
if sys.argv[1:2]==['pull'] and os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',sys.argv[2]]:sys.exit(1)
|
||||
''')
|
||||
def run(args,extra=None):
|
||||
return subprocess.run(args,cwd=project,env={**environment,**(extra or {})},capture_output=True,text=True)
|
||||
def task():return ['bash',str(project/'scripts/update-infrastructure.sh')]
|
||||
def recorded():return [json.loads(line) for line in calls.read_text().splitlines()]
|
||||
for shell in ['bash','pwsh']:
|
||||
for image,enabled,expected in [('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','true','latest'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.13.1','true','latest'),('customer.invalid/watcher:1.22.3','true','1.22.3'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','false','1.22.3')]:
|
||||
(project/'.env').write_text(f'WATCHTOWER_IMAGE={image}\nINFRASTRUCTURE_AUTO_UPDATE_ENABLED={enabled}\nKEEP=original\n')
|
||||
args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
|
||||
p=run(args);assert p.returncode==0,p.stderr
|
||||
content=(project/'.env').read_text();assert f'WATCHTOWER_IMAGE={image.rsplit(":",1)[0]}:{expected}' in content,content
|
||||
assert 'KEEP=original' in content
|
||||
assert run(args).returncode==0 and (project/'.env').read_text()==content
|
||||
for shell in ['bash','pwsh']:
|
||||
(project/'.env').write_text('WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest\nWATCHTOWER_UPDATE_TARGET=easyai-updater\n')
|
||||
args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
|
||||
assert run(args,{'UPDATE_SERVICES':'easyai-server'}).returncode==0
|
||||
assert 'WATCHTOWER_SCOPE=' not in (project/'.env').read_text()
|
||||
assert run(args,{'UPDATE_SERVICES':''}).returncode==0
|
||||
assert 'WATCHTOWER_SCOPE=easyai' in (project/'.env').read_text()
|
||||
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
|
||||
install=['bash',str(project/'scripts/install-infrastructure-update-task.sh'),str(project)]
|
||||
assert run(install).returncode==0
|
||||
first=cron.read_text();assert 'existing-backup' in first and first.count('# easyai-infrastructure-update-')==1
|
||||
assert '\\%' in first and 'project\\ space' in first,first
|
||||
assert run(install).returncode==0 and cron.read_text()==first
|
||||
assert run(install,{'INFRA_TEST_CRON_ERROR':'1'}).returncode!=0 and cron.read_text()==first
|
||||
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false\n')
|
||||
assert run(install).returncode==0 and 'easyai-infrastructure-update' not in cron.read_text() and 'existing-backup' in cron.read_text()
|
||||
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
|
||||
calls.write_text('');assert run(task()).returncode==0
|
||||
assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded()
|
||||
assert ['compose','pull','easyai-updater'] in recorded() and ['compose','up','-d','--no-deps','easyai-updater'] in recorded()
|
||||
assert not any('mongo' in args for args in recorded())
|
||||
calls.write_text('');assert run(task(),{'INFRA_TEST_PULL_FAIL':'1'}).returncode!=0
|
||||
assert not any('up' in args for args in recorded());assert not (project/'data/infrastructure-update/lock').exists()
|
||||
calls.write_text('');p=subprocess.Popen(task(),cwd=project,env={**environment,'INFRA_TEST_SLEEP':'1'},stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||||
for _ in range(100):
|
||||
if (project/'data/infrastructure-update/lock/pid').exists():break
|
||||
time.sleep(.01)
|
||||
assert run(task()).returncode==0
|
||||
assert p.wait(timeout=5)==0 and sum('pull' in args for args in recorded())==2
|
||||
calls.write_text('');assert run(task(),{'INFRA_TEST_V1':'1'}).returncode==0
|
||||
assert ['v1','up','-d','--no-deps','watchtower'] in recorded() and ['v1','up','-d','--no-deps','easyai-updater'] in recorded()
|
||||
calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)])
|
||||
assert p.returncode==0,p.stderr
|
||||
assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded(),recorded()
|
||||
calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)],{'INFRA_TEST_PULL_FAIL':'1'})
|
||||
assert p.returncode!=0 and not any('up' in args for args in recorded())
|
||||
for shell in ['bash','pwsh']:
|
||||
args=task() if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)]
|
||||
for transport in ['', '1']:
|
||||
kind='v1' if transport else 'compose'
|
||||
for failed,other in [('watchtower','easyai-updater'),('easyai-updater','watchtower')]:
|
||||
calls.write_text('');p=run(args,{'INFRA_TEST_PULL_FAIL':failed,'INFRA_TEST_V1':transport})
|
||||
assert p.returncode!=0,(shell,p.stderr)
|
||||
assert [kind,'pull',failed] in recorded() and [kind,'pull',other] in recorded()
|
||||
assert [kind,'up','-d','--no-deps',failed] not in recorded()
|
||||
assert [kind,'up','-d','--no-deps',other] in recorded(),(shell,recorded())
|
||||
calls.write_text('');p=run(args,{'INFRA_TEST_LEGACY':'1','INFRA_TEST_V1':transport})
|
||||
assert p.returncode==0 and not any('easyai-updater' in a for a in recorded()),(shell,p.stderr,recorded())
|
||||
calls.write_text('');p=run(args,{'INFRA_TEST_NO_HEALTH':'1'})
|
||||
assert p.returncode==0,(shell,p.stderr)
|
||||
assert any(a[:3]==['exec','easyai-updater-test-id','node'] for a in recorded())
|
||||
print(json.dumps({'passed':True,'cases':['Bash/PowerShell default migration and explicit choices','cron registration deduplication and existing task preservation','cron read failure leaves jobs untouched','Bash/PowerShell independent component maintenance and partial failure reporting','legacy profile skips updater','concurrent task excluded','Compose v1 compatibility','only Watchtower and updater updated','service health probes'],'durationSeconds':round(time.monotonic()-started,3)}))
|
||||
@@ -34,6 +34,8 @@ def fixture(base):
|
||||
shutil.copytree(path, root / name)
|
||||
else:
|
||||
shutil.copy2(path, root / name)
|
||||
(root / 'scripts/install-infrastructure-update-task.sh').write_text('#!/bin/sh\nexit 0\n')
|
||||
(root / 'scripts/Install-InfrastructureUpdateTask.ps1').write_text('# Scheduler transport is verified separately.\n')
|
||||
(root / 'data').mkdir()
|
||||
(root / '.env').write_text('SYSTEM_UPDATE_MODE=legacy_watchtower\nNUXT_PUBLIC_BASE_APIURL=http://127.0.0.1:3001\nCONFIG_JWT_SECRET=existing-jwt-secret-that-is-long-enough\nWS_AUTH_WS_TICKET_SECRET=existing-ticket-secret-that-is-long-enough\n')
|
||||
(root / '.gitignore').write_text('.env\n*.log\n')
|
||||
|
||||
@@ -26,7 +26,7 @@ grep -qx 'COMPOSE_PROFILES=customer-profile' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'WATCHTOWER_CLEANUP=' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'WATCHTOWER_HTTP_API_PERIODIC_POLLS=false' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'WATCHTOWER_UPDATE_TARGET=easyai-server easyai-wsgateway easyai-web' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'MODEL_CONVERSION_WORKER_VERSION=latest' "$LEGACY_ENV_FILE"
|
||||
grep -qx 'MODEL_CONVERSION_HUNYUAN_ENABLED=false' "$LEGACY_ENV_FILE"
|
||||
cp "$LEGACY_ENV_FILE" "$TMP_DIR/legacy-first-run.env"
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
# 两种更新模式均启用内网接口；安全令牌由安全初始化脚本生成。
|
||||
WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3
|
||||
WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# 宿主机定时维护更新组件，保留客户明确关闭的选择。
|
||||
INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
cd "$PROJECT_ROOT"
|
||||
. ./scripts/init-security-env.sh
|
||||
[ "$(read_env_value .env INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
|
||||
state="$PROJECT_ROOT/data/infrastructure-update"
|
||||
mkdir -p "$state"
|
||||
lock="$state/lock"
|
||||
if ! mkdir "$lock" 2>/dev/null; then
|
||||
pid="$(cat "$lock/pid" 2>/dev/null || true)"
|
||||
if [[ "$pid" =~ ^[0-9]+$ ]] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
rm -f "$lock/pid"
|
||||
rmdir "$lock" 2>/dev/null || exit 0
|
||||
mkdir "$lock" 2>/dev/null || exit 0
|
||||
else
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
printf '%s\n' "$$" > "$lock/pid"
|
||||
trap 'rm -f "$lock/pid"; rmdir "$lock" 2>/dev/null || true' EXIT
|
||||
exec > "$state/last-run.log" 2>&1
|
||||
printf '%s 检查 Watchtower 和 updater 更新\n' "$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
registry_config="$(read_env_value .env DOCKER_CONFIG_FILE)"
|
||||
docker_options=()
|
||||
if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then
|
||||
docker_options=(--config "$(dirname "$registry_config")")
|
||||
fi
|
||||
if docker "${docker_options[@]}" compose version >/dev/null 2>&1; then
|
||||
compose() { docker "${docker_options[@]}" compose "$@"; }
|
||||
else
|
||||
compose() {
|
||||
if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then
|
||||
DOCKER_CONFIG="$(dirname "$registry_config")" docker-compose "$@"
|
||||
else
|
||||
docker-compose "$@"
|
||||
fi
|
||||
}
|
||||
fi
|
||||
available="$(compose config --services)"
|
||||
printf '%s\n' "$available" | grep -qx watchtower || { echo '未配置 Watchtower'; exit 1; }
|
||||
maintain_service() {
|
||||
local service="$1" container_id status attempt
|
||||
echo "检查 $service"
|
||||
# A failed service must not block the other update channel.
|
||||
compose pull "$service" || return 1
|
||||
compose up -d --no-deps "$service" || return 1
|
||||
for ((attempt=0; attempt<12; attempt++)); do
|
||||
container_id="$(compose ps -q "$service")" || return 1
|
||||
if [ -n "$container_id" ]; then
|
||||
status="$(docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")" || return 1
|
||||
if [ "$status" = 'true healthy' ]; then
|
||||
echo "$service 已恢复可用"
|
||||
return 0
|
||||
fi
|
||||
if [ "$status" = 'true none' ]; then
|
||||
if [ "$service" = watchtower ]; then
|
||||
if docker exec "$container_id" /watchtower --health-check; then return 0; fi
|
||||
else
|
||||
if docker exec "$container_id" node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))"; then return 0; fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
echo "$service 暂未恢复可用" >&2
|
||||
return 1
|
||||
}
|
||||
failed=0
|
||||
for service in watchtower easyai-updater; do
|
||||
# config --services respects active profiles: legacy deployments stay legacy.
|
||||
if ! printf '%s\n' "$available" | grep -qx "$service"; then
|
||||
echo "$service 未启用，跳过维护"
|
||||
continue
|
||||
fi
|
||||
if ! maintain_service "$service"; then
|
||||
echo "$service 本次维护未成功，下次定时任务将继续检查" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
exit "$failed"
|
||||
@@ -429,6 +429,7 @@ function Main {
|
||||
|
||||
. (Join-Path $script:Root "scripts\Initialize-SecurityEnv.ps1")
|
||||
Initialize-SecurityEnv -Path (Join-Path $script:Root ".env") -Mode $script:SecurityEnvMode
|
||||
& (Join-Path $script:Root "scripts/Initialize-InfrastructureUpdateEnv.ps1") -ProjectRoot $script:Root
|
||||
|
||||
. (Join-Path $script:Root "scripts\Initialize-PublicApiBaseUrl.ps1")
|
||||
$publicUrlMode = if ($script:SkipDeployQuestions) { "upgrade" } else { "configure" }
|
||||
@@ -442,9 +443,11 @@ function Main {
|
||||
|
||||
if ($script:DeployDryRun) {
|
||||
Warn "dry-run mode: skip docker and services"
|
||||
& (Join-Path $script:Root "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $script:Root -DryRun
|
||||
} else {
|
||||
Test-Docker
|
||||
Start-Services
|
||||
& (Join-Path $script:Root "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $script:Root
|
||||
}
|
||||
|
||||
$ip = if ($script:DeployIP) { $script:DeployIP } else { "127.0.0.1" }
|
||||
|
||||
@@ -648,6 +648,7 @@ main() {
|
||||
# shellcheck source=scripts/init-security-env.sh
|
||||
. ./scripts/init-security-env.sh
|
||||
init_security_env .env "$SECURITY_ENV_MODE"
|
||||
bash ./scripts/init-infrastructure-update-env.sh "$PWD"
|
||||
|
||||
# 在拉取镜像或重建服务前完成公开地址迁移；非法或无法推导时立即终止。
|
||||
# shellcheck source=scripts/init-public-api-base-url.sh
|
||||
@@ -666,9 +667,11 @@ main() {
|
||||
echo ""
|
||||
echo "⚠️ dry-run 模式：跳过 Docker 安装和服务启动"
|
||||
echo " 配置文件已生成，可直接运行 ./start.sh 完成部署"
|
||||
bash ./scripts/install-infrastructure-update-task.sh "$PWD" --dry-run
|
||||
else
|
||||
install_docker
|
||||
start_services
|
||||
bash ./scripts/install-infrastructure-update-task.sh "$PWD"
|
||||
configure_http_nginx_proxy
|
||||
export DEPLOY_DOMAIN
|
||||
run_https_setup
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# updater 双保险维护验收
|
||||
|
||||
日期：2026-10-08。代码位于 `/Users/wangbo/projects/easyai`。本记录补充前序 Watchtower 宿主机维护验收，描述本轮新增的 updater 独立定时更新通道。
|
||||
|
||||
## 结果
|
||||
|
||||
宿主机定时任务每六小时分别维护 Watchtower 和已启用的 updater；原 Watchtower 每 24 小时维护 updater 的机制保留。无需新增容器或再注册另一条宿主机任务。
|
||||
|
||||
每项维护分别拉取、按需重建和验证健康状态。某项失败仍继续另一项，同时记录失败并返回非零结果。不会因 Watchtower 不可用而跳过 updater。旧部署未启用 updater 时保持原更新模式。
|
||||
|
||||
验收时尚未提交或推送源码；本轮真实运行结果不代表已部署至线上。本轮没有实际注册本机 cron 或 Windows 计划任务。旧机器需由运维执行一次新版部署脚本接入；发布源码本身不能给机器安装宿主机任务。
|
||||
|
||||
## 修改前真实复现
|
||||
|
||||
独立 Compose 项目 `easyai-updater-insurance-20261008`，Watchtower 1.22.3、updater 1.1.0，本机 registry 发布 updater 1.2.0。主动停止本次自建的 Watchtower，并把该项目 Watchtower 镜像设置为不存在的标签，模拟 Watchtower 恢复失败。
|
||||
|
||||
分别运行原 Bash 和 PowerShell 定时入口：均在 Watchtower 拉取失败时退出；updater 继续运行旧版本 1.1.0，容器及镜像均未变化。
|
||||
|
||||
证据：`before.json`、`before-bash.log`、`before-powershell.log`，原入口快照保存在 `before-scripts/`。这说明原定时维护只负责 Watchtower，不能在其失效时直接维护 updater。
|
||||
|
||||
## 本轮变更及影响范围
|
||||
|
||||
- 修改 `scripts/update-infrastructure.sh` 和 `scripts/Update-Infrastructure.ps1`：独立处理两个组件，失败分别记录，最后汇总结果；保持部署目录、Docker 登录配置、并发锁及只重建指定服务的行为。
|
||||
- 修改 Bash/PowerShell 任务安装器的提示文案，保留原任务标识、每六小时周期、去重及关闭方式，不重复注册任务。
|
||||
- 更新 `.env.sample` 和 README，说明 updater 两条维护路径。
|
||||
- 扩展 `scripts/test-infrastructure-auto-update.py`，覆盖两种入口、两种 Compose 命令路径、独立失败、legacy 跳过及健康检查。
|
||||
|
||||
主要调用方仍为 `start.sh/update.sh/start.ps1/update.ps1` 安装的定时任务。没有修改业务自动更新开关，也没有改动 updater、后端、前端业务代码及已有任务存储格式。此前工作区中其他未提交改动均保留。
|
||||
|
||||
## 实际运行结果
|
||||
|
||||
| 场景 | 实际结果 |
|
||||
| --- | --- |
|
||||
| Watchtower 已停止且拉取失败，Bash 定时入口 | updater 实际升级 1.1.0 → 1.2.0；Watchtower 仍停止，证明该升级不依赖它；返回失败以保留 Watchtower 故障结果 |
|
||||
| 相同条件，PowerShell 定时入口 | updater 实际升级 1.1.0 → 1.2.0；同样独立执行并报告部分失败 |
|
||||
| updater 历史任务 | Bash 独立更新后，实际任务 API 仍能读取原有成功任务及其消息 |
|
||||
| updater 拉取失败 | updater 容器未停止或重建；Watchtower 仍恢复健康；整体报告失败 |
|
||||
| 正常维护及镜像无变化 | 两组件均健康；镜像和配置无变化时容器 ID 不变 |
|
||||
| legacy profile 未启用 updater | Bash、PowerShell 均不创建或启动 updater |
|
||||
| 原 Watchtower 更新通道 | 重新开启轮询并发布 updater 新标签，Watchtower 仍实际更新 updater 1.1.0 → 1.2.0 |
|
||||
| 业务服务对照 | 全程容器 ID 不变，没有被维护入口直接重建 |
|
||||
|
||||
实际运行验收耗时 27.58 秒，不含镜像准备及修改前复现。证据：`after.json`、各场景 `.log`、`watchtower-final.log`。这是自建 registry、真实 Compose 和真实更新组件的运行结果。业务服务为独立 Node 对照容器，并未部署整套业务及数据库。
|
||||
|
||||
## 定向检查
|
||||
|
||||
- `python3 scripts/test-infrastructure-auto-update.py`：通过，19.496 秒。覆盖 Bash/PowerShell、Compose v2/v1 命令路径、单项失败不影响另一项、全部拉取失败不重建、legacy 跳过、并发排除、健康探测，以及原有迁移和 cron 去重保护。
|
||||
- `pwsh -NoProfile -File scripts/Test-InfrastructureUpdateTask.ps1`：通过，0.18 秒。验证注册参数及去重、关闭行为；计划任务 API 为测试替身。
|
||||
- Bash/PowerShell 语法、ShellCheck、diff 空白检查：通过，合计 0.72 秒。PowerShell UTF-8 BOM 保持不变，供 Windows PowerShell 5.1 使用。
|
||||
|
||||
证据：`validation.json`、`quality-checks.json` 及本轮工具输出。没有运行全量前端质量门禁，本轮未涉及前端业务代码。
|
||||
|
||||
## 验证边界
|
||||
|
||||
Windows 宿主机真实计划任务注册、生产六小时和 24 小时长周期、客户私有仓库网络及权限尚未验证。Compose v1 是命令适配行为验收，实际容器验收使用 Docker Compose v5.3.1。PowerShell 实际容器入口在 macOS 上运行。
|
||||
|
||||
历史终态任务持久化已实际验证；本轮没有在业务更新执行到一半时替换 updater 的故障演练，也不据此宣称所有活动任务恢复已验收。
|
||||
|
||||
Compose 根据启用的 profile 选择可维护服务，运行 `config --services` 已实测关闭 profile 时不包含 updater。避免直接显式启动未启用的 updater，因显式指定服务会自动启用其 profile。[Docker 官方说明](https://docs.docker.com/compose/how-tos/profiles/)
|
||||
|
||||
本方案仍需 Docker 和镜像仓库可用；镜像启动失败不会自动回滚。每项维护有健康等待并报告失败，后续定时任务继续检查。
|
||||
|
||||
本次已清理四个自建验收容器及独立网络，原本机 `test-watchtower-1` 保持健康。证据：`cleanup.json`。
|
||||
|
||||
原始运行日志与入口快照保存在本机证据目录：`/Users/wangbo/.codex/visualizations/2026/10/07/01a11505-cae4-76e0-9ba9-235f6962b2b4/updater-double-insurance-20261008`。本目录附带修改前、修改后及定向检查的结构化结果。
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"passed": true,
|
||||
"bashUpdatedUpdaterWithWatchtowerStoppedAndPullFailed": true,
|
||||
"powershellUpdatedUpdaterWithWatchtowerStoppedAndPullFailed": true,
|
||||
"failedServiceReportedWhileOtherServiceMaintained": true,
|
||||
"persistedJobRetained": true,
|
||||
"updaterPullFailurePreservedContainerAndWatchtowerRecovered": true,
|
||||
"unchangedImagesNoRecreate": true,
|
||||
"legacyProfileDoesNotStartUpdater": true,
|
||||
"originalWatchtowerChannelStillUpdatesUpdater": true,
|
||||
"businessContainerUnchanged": true,
|
||||
"seconds": 27.58,
|
||||
"before": [
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-watchtower",
|
||||
"id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c",
|
||||
"image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7",
|
||||
"running": false
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-updater",
|
||||
"id": "9837e6b0328798dbbad17ff310ef8d6d27761c1d152d374a91e56fdb3009c31e",
|
||||
"image": "sha256:275e9ff7fb453d16feabfb9b92e1cd603f49c434bf2daa80a9f5ad6f505bffcb",
|
||||
"running": true
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-business",
|
||||
"id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f",
|
||||
"image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493",
|
||||
"running": true
|
||||
}
|
||||
],
|
||||
"afterIndependentTimer": [
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-watchtower",
|
||||
"id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c",
|
||||
"image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7",
|
||||
"running": false
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-updater",
|
||||
"id": "f4b5029ae1bd5b9d33ef472039b3e938242ac68c5d3a36019035a745532a0734",
|
||||
"image": "sha256:d7e4d04d4e614d537a08ee24f58317a3d6734e3514ca61eafc92bd1bb7a8d6ef",
|
||||
"running": true
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-business",
|
||||
"id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f",
|
||||
"image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493",
|
||||
"running": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"reproduced": true,
|
||||
"watchtowerStopped": true,
|
||||
"watchtowerPullFailed": true,
|
||||
"updaterRemainedOld": true,
|
||||
"bashExitCode": 1,
|
||||
"powershellExitCode": 1,
|
||||
"containers": [
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-watchtower",
|
||||
"id": "f177e0655e789190f9f06533e1035d166b5924db7d2afc1d1e242abb43b1bd0c",
|
||||
"image": "sha256:6ba1e78a0c9ce1ae5389ecf8a78426b82beaf3357c71d52548f7d05fd92eebb7",
|
||||
"running": false
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-updater",
|
||||
"id": "9837e6b0328798dbbad17ff310ef8d6d27761c1d152d374a91e56fdb3009c31e",
|
||||
"image": "sha256:275e9ff7fb453d16feabfb9b92e1cd603f49c434bf2daa80a9f5ad6f505bffcb",
|
||||
"running": true
|
||||
},
|
||||
{
|
||||
"name": "/easyai-updater-insurance-20261008-business",
|
||||
"id": "6b733cf4ee75e2136430bb1b53ec49c1685d3958ad7cebe862b4a52b139e847f",
|
||||
"image": "sha256:5bbd38fd80132923f2ea205d02836fefdf7eb3555fd93f0bb58e26a899c0e493",
|
||||
"running": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"ownContainersRemoved": [
|
||||
"/easyai-updater-insurance-20261008-watchtower",
|
||||
"/easyai-updater-insurance-20261008-updater",
|
||||
"/easyai-updater-insurance-20261008-business",
|
||||
"/easyai-updater-insurance-20261008-registry"
|
||||
],
|
||||
"ownNetworkRemoved": true,
|
||||
"originalLocalTestWatchtowerHealthy": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"BashSyntax": true,
|
||||
"PowerShellSyntax": true,
|
||||
"ShellCheck": true,
|
||||
"DiffWhitespace": true,
|
||||
"seconds": 0.72
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"infrastructureTests": {
|
||||
"passed": true,
|
||||
"seconds": 19.496
|
||||
},
|
||||
"powershellTaskContract": {
|
||||
"passed": true,
|
||||
"seconds": 0.18
|
||||
},
|
||||
"windowsPowerShellEncoding": {
|
||||
"bomPreserved": true
|
||||
},
|
||||
"fullQualityGateRun": false
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
# 在线更新发布源修复与测试验收（2026-10-07）
|
||||
|
||||
本次修复已在 `https://demo.51easyai.com` 完成真实浏览器验收。候选镜像和测试发布源只用于测试环境；验收时尚未提交代码；该测试验收未更新正式 `latest` 镜像和仓库根目录的正式发布清单。
|
||||
|
||||
## 已复现问题与根因
|
||||
|
||||
1. 测试站运行 `3.22.0.20260930.3`，页面从官方站运行版本接口检测到 `3.23.1.20261006`。
|
||||
2. 浏览器点击“立即更新”，任务 `bcaa8c58-3628-42fc-af02-bc5f97bbc1c2` 返回 `no_update`，实际版本和核心容器均未改变；页面仍提示有新版本。
|
||||
3. 更新器实际读取 `/var/lib/easyai-updater/release-manifest.json`，该路径由 `data/deployment-config/legacy-compose.json` 覆盖，内容仍指向旧版镜像。更新器不获取远程发布清单。
|
||||
4. 页面检测来源与执行来源不同；前端没有核对任务实际提供的版本，误把旧清单的“无需更新”作为新版已经完成更新。
|
||||
5. 正式部署仓库的发布清单也仍为 `3.22.0.20261002.2`；镜像仓库有后端 `3.23.1.20261006` 标签，但没有同版本前端标签。单纯改成本地另一条清单路径不能解决发布信息不一致。
|
||||
|
||||
## 最小修复及兼容性
|
||||
|
||||
- 更新器默认从部署仓库 `main` 分支获取正式清单；私有源使用 `UPDATE_MANIFEST_URL`，离线部署须显式选择 `UPDATE_MANIFEST_SOURCE=local`。默认远程模式忽略历史本地路径覆盖，获取失败不会退回旧清单。
|
||||
- 受控模式的版本检测通过更新器获取可安装版本，不再取官方站当前运行版本；旧 Watchtower 模式保留原协议和检测缓存。
|
||||
- 新版客户端提交检测到的版本与发布标识；执行前重新获取清单，目标发生变化时拒绝安装另一版本。旧客户端只传 `action` 的请求继续兼容。
|
||||
- 任务保存完整清单，恢复及回滚使用同一快照。前端核对成功或“无需更新”的实际版本，避免错误提示；普通更新、强制更新和管理员新版提醒成功后均刷新检测结果。
|
||||
- 只对更高版本显示新版提示；受控更新拒绝自动回退到更低版本，包括强制更新。发布源不可用时显示“暂时无法检查”和“重新检查”。
|
||||
- 主要调用方：系统设置更新卡片、管理员新版提醒、任务恢复入口、自动更新调度及旧手动接口；未增加业务 Provider 或改变自动更新开关。
|
||||
|
||||
责任代码与本次改动位置：
|
||||
|
||||
- 后端 `apps/easyai-updater/src/release-source.ts`、`update-runner.ts`、`main.ts`：发布源、目标固定、快照恢复与内部发布信息接口。
|
||||
- 后端 `apps/easyai-server-main/src/app.service.ts`、`system-update/managed-update.service.ts`、`system-update/system-update.controller.ts`：统一检测来源、兼容请求及目标字段传递。
|
||||
- 前端 `composables/system-update/useSystemUpdateJob.ts`：核对更新结果与检测目标；`pages/admin/sys/operationConfig.vue` 和 `components/product-update/ProductUpdateAdminTouchpoint.vue`：版本比较、检测异常提示及成功后刷新。
|
||||
- 部署仓库 `docker-compose.yml`、`.env.sample`：默认远程发布源与显式离线选项；原正式清单保留不变。
|
||||
|
||||
## 真实运行验收
|
||||
|
||||
先部署保留旧版本号的修复版控制入口作为测试基线；前端基线使用独立镜像，使实际任务能够验证网页容器替换。最终安装的镜像来自本次工作区构建。候选清单见同目录 `release-manifest.candidate.json`，不作为正式发布清单。
|
||||
|
||||
| 场景 | 结果 |
|
||||
| --- | --- |
|
||||
| 管理员通过真实登录界面进入系统设置并点击立即更新 | 只提交一次请求；任务 `5a2968c5-9615-4e1f-9d88-79ada0f6aa06` 成功，页面约 88.9 秒完成反馈 |
|
||||
| 版本与容器核对 | `3.22.0.20260930.3` → `3.24.0.20261007`；后端和网页容器实际替换，实时服务镜像无需变化 |
|
||||
| 页面刷新 | 显示 `3.24.0.20261007`、已是最新，不重复弹出新版提醒 |
|
||||
| 同版本普通更新 | 返回 `no_update`，不重复替换容器 |
|
||||
| 强制更新确认后取消 | 不创建任务 |
|
||||
| 发布源真实返回 HTTP 503 | 页面显示暂时无法检查，不能误报已是最新；恢复源后点击重新检查恢复正常 |
|
||||
| 发布源改成旧正式清单 | 不提示有新版；点击强制更新后，任务 `92182a3e-1948-4d82-99bc-aacb7c1c601f` 在镜像拉取前被拒绝，原因 `release_older_than_current` |
|
||||
| 配置保留 | 核心服务环境变量、完整容器配置（除镜像）、端口、挂载路径及权限、网络地址和别名保持一致；宿主机 `.env` 与 `docker-compose.yml` 哈希不变 |
|
||||
| 自动更新设置 | 验收前后均为开启，未改变原开关 |
|
||||
| 更新器实际重启 | 已完成任务完整保留，恢复后仍提供同一测试发布信息；四个相关容器均健康 |
|
||||
|
||||
首次候选镜像补装运行依赖时移除了内部共享包，启动检查失败后恢复了原服务。补齐共享包并通过加载检查后，重新部署健康基线，再进行上述正式浏览器验收；没有把失败启动计为升级成功。
|
||||
|
||||
## 定向检查
|
||||
|
||||
本地检查使用 Node `22.22.0`、Yarn `1.22.22`，共 79 个不同的定向行为用例通过：
|
||||
|
||||
| 检查 | 结果与耗时 |
|
||||
| --- | --- |
|
||||
| updater Node 测试 | 14 个通过，约 1.6 秒；覆盖真实 HTTP 发布源、源失败、目标变化、快照恢复及旧版回退保护 |
|
||||
| 后端 system-update Jest | 38 个通过，约 2.3 秒；包含新旧检测协议、实际 DTO 校验、自动更新调度和旧手动接口 |
|
||||
| 前端任务与实际挂载测试 | 首轮 13 个通过，约 49 秒；追加后的设置与更新处理函数 5 个通过，约 4 秒，其中新增强制更新用例 1 个 |
|
||||
| 版本工具和产品版本测试 | 13 个通过，约 0.13 秒 |
|
||||
| 部署脚本隔离 dry-run | 通过，约 4.3 秒，确认默认远程源及原部署契约 |
|
||||
| 后端及更新器构建 | 通过，约 25 秒 |
|
||||
| 最终前端生产构建 | 通过，约 168 秒 |
|
||||
|
||||
隔离环境中的旧实现会失败：前端将旧版 `no_update` 误判成功；更新器六个新增发布源场景不能满足预期。相同失败用例在修复后通过。HTTP 发布源测试使用受控 Docker 替身，真实容器替换另外通过测试站验收；没有用源码断言或构建替代运行验收。
|
||||
|
||||
## 证据与测试环境维护
|
||||
|
||||
完整本地证据目录：`/Users/wangbo/.codex/visualizations/2026/10/07/01a11505-cae4-76e0-9ba9-235f6962b2b4/system-update-diagnosis`。
|
||||
|
||||
- `acceptance-summary.json`、`browser-upgrade-acceptance.json`、`browser-acceptance.json`：实际请求、任务、版本及异常场景。
|
||||
- `acceptance-update-success.png`、`acceptance-final-card.png`、`acceptance-feed-unavailable.png`、`acceptance-older-release-blocked.png`：真实页面结果。
|
||||
- `config-before-browser.json`、`config-after-browser.json`、`mount-verification.json`、`final-verification.json`：配置、健康及任务持久化核对。Docker 返回的挂载数组顺序不稳定，已按内容核对路径和权限一致。
|
||||
- `*-tests.log`、`*-build*.log`、`updater-before-tests.log`、`frontend-before-tests.log`：定向检查与修复前失败记录。
|
||||
|
||||
测试环境保留内部测试发布源容器 `easyai-update-release-feed`，无新增宿主机端口。任务专用配置位于 `/root/easyai/data/update-acceptance-20261007`，其中 `accepted.json` 和更新后的 `baseline.json` 指向实际验收镜像；`rollback.json` 保留原镜像引用。维护测试站时须使用原 Compose、原 legacy override 与该目录下 `accepted.json` 的组合，不能将该测试清单误发布成正式清单。
|
||||
|
||||
## 未验证范围及正式发布条件
|
||||
|
||||
- 未发布生产，未执行全站业务回归或全量质量门禁；健康检查和更新入口验收不能代表全部业务兼容性。
|
||||
- 定时自动更新实际到点执行未验收，调度逻辑已做定向测试；自动更新设置保持原值。
|
||||
- 已测试持久化快照恢复，并实际重启验证已完成任务；没有在真实容器替换过程中故意中断 updater。旧 updater 未带清单快照的在途任务跨版本迁移尚未验收，升级更新器前应先确认没有执行中的更新任务。
|
||||
- 正式发布时须一起交付 updater 1.2.0、后端和前端，并在镜像完整发布后同步正式清单的版本、发布标识、不可变摘要及大小。只改官方站版本或只推送 `latest` 仍无法代表客户可安装的新版本。
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"channel": "stable",
|
||||
"releaseId": "update-source-acceptance-20261007-r3",
|
||||
"productVersion": "3.24.0.20261007",
|
||||
"platform": "linux/amd64",
|
||||
"publishedAt": "2026-10-07T06:45:45.630545+00:00",
|
||||
"managedUpdateAllowed": true,
|
||||
"scheduledUpdateAllowed": true,
|
||||
"rollbackSafe": true,
|
||||
"minimumUpdaterVersion": "1.2.0",
|
||||
"minimumDeploymentSchemaVersion": 2,
|
||||
"requiredFreeBytes": 8589934592,
|
||||
"components": [
|
||||
{
|
||||
"key": "server-main",
|
||||
"image": "registry.cn-shanghai.aliyuncs.com/comfy-ai/comfy-server",
|
||||
"digest": "sha256:b334362777cfc2a1064a264cf10dac2766b32674627d98775be8a2d9ceb7da5b",
|
||||
"imageSizeBytes": 2747248955,
|
||||
"buildId": "test-source-20261007"
|
||||
},
|
||||
{
|
||||
"key": "ws-gateway",
|
||||
"image": "registry.cn-shanghai.aliyuncs.com/easyaigc/wsgateway",
|
||||
"digest": "sha256:a2b5fb67384c7f0eca7c7a768b5deacb4c0d2195dfbeb6e16ef40d6dc097bf25",
|
||||
"imageSizeBytes": 1982472178,
|
||||
"buildId": "test-source-20261007"
|
||||
},
|
||||
{
|
||||
"key": "web",
|
||||
"image": "registry.cn-shanghai.aliyuncs.com/comfy-ai/one-ai",
|
||||
"digest": "sha256:9ea88a929e743dcba2c7a38402c913d011af99b3ccc63a9cf8b4f97e4abda4d5",
|
||||
"imageSizeBytes": 886998757,
|
||||
"buildId": "test-source-20261007"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -215,6 +215,7 @@ if (-not (Get-EnvValue $environmentContent "SYSTEM_UPDATE_MODE")) {
|
||||
& (Join-Path $scriptDir "scripts\Apply-UpdateEnvMigrations.ps1") `
|
||||
-EnvPath (Join-Path $scriptDir ".env") `
|
||||
-ProjectRoot $scriptDir
|
||||
& (Join-Path $scriptDir "scripts/Initialize-InfrastructureUpdateEnv.ps1") -ProjectRoot $scriptDir
|
||||
|
||||
. (Join-Path $scriptDir "scripts\Initialize-ServerHttpBindIp.ps1")
|
||||
Initialize-ServerHttpBindIp -Path (Join-Path $scriptDir ".env")
|
||||
@@ -390,6 +391,8 @@ if ($hasComposeV2) {
|
||||
|
||||
Wait-CoreServices
|
||||
|
||||
& (Join-Path $scriptDir "scripts/Install-InfrastructureUpdateTask.ps1") -ProjectRoot $scriptDir
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "================================"
|
||||
Write-Host " 更新完成"
|
||||
|
||||
@@ -180,6 +180,7 @@ init_server_http_bind_ip .env
|
||||
# shellcheck source=scripts/init-security-env.sh
|
||||
. ./scripts/init-security-env.sh
|
||||
init_security_env .env upgrade
|
||||
bash ./scripts/init-infrastructure-update-env.sh "$SCRIPT_DIR"
|
||||
|
||||
if [ -n "${UPDATE_RELEASE_VERSION:-}" ]; then
|
||||
write_env_value .env VERSION "$UPDATE_RELEASE_VERSION"
|
||||
@@ -337,4 +338,5 @@ fi
|
||||
if ! wait_for_core_services; then
|
||||
exit 1
|
||||
fi
|
||||
bash ./scripts/install-infrastructure-update-task.sh "$SCRIPT_DIR"
|
||||
echo "🎉 EasyAI 应用更新成功"
|
||||
|
||||
Reference in New Issue
Block a user