feat(deploy): 为更新组件增加定时维护与双保险

scripts/Initialize-InfrastructureUpdateEnv.ps1

@@ -0,0 +1,20 @@
+param([Parameter(Mandatory=$true)][string]$ProjectRoot)
+$ErrorActionPreference = 'Stop'
+. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
+$path = Join-Path $ProjectRoot '.env'
+$content = [System.IO.File]::ReadAllText($path)
+if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return }
+$image = Get-EnvValue $content 'WATCHTOWER_IMAGE'
+if ([string]::IsNullOrWhiteSpace($image) -or $image -match '^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:\d+\.\d+\.\d+$') {
+    $content = Set-EnvValue $content 'WATCHTOWER_IMAGE' 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest'
+    [System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false))
+}
+
+# Preserve old scope during a targeted update; unchanged containers keep their labels.
+$image = Get-EnvValue $content 'WATCHTOWER_IMAGE'
+$target = Get-EnvValue $content 'WATCHTOWER_UPDATE_TARGET'
+$scope = Get-EnvValue $content 'WATCHTOWER_SCOPE'
+if ([string]::IsNullOrWhiteSpace($env:UPDATE_SERVICES) -and [string]::IsNullOrWhiteSpace($scope) -and $image -eq 'registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' -and $target -in @('', 'easyai-updater', 'easyai-server easyai-wsgateway easyai-web')) {
+    $content = Set-EnvValue $content 'WATCHTOWER_SCOPE' 'easyai'
+    [System.IO.File]::WriteAllText($path, $content, [System.Text.UTF8Encoding]::new($false))
+}

scripts/Install-InfrastructureUpdateTask.ps1

@@ -0,0 +1,30 @@
+param(
+    [Parameter(Mandatory=$true)][string]$ProjectRoot,
+    [switch]$DryRun
+)
+$ErrorActionPreference = 'Stop'
+$ProjectRoot = (Resolve-Path $ProjectRoot).Path
+. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
+$enabled = (Get-EnvValue ([System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env'))) 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -ne 'false'
+$hash = [System.Security.Cryptography.SHA256]::Create()
+try { $id = ([System.BitConverter]::ToString($hash.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($ProjectRoot)))).Replace('-', '').Substring(0, 12) }
+finally { $hash.Dispose() }
+$name = "EasyAI-Watchtower-$id"
+if ($DryRun -and -not $enabled) { Write-Host "更新组件定时任务（dry-run）：已关闭"; return }
+if ($DryRun) { Write-Host "Watchtower 和 updater 定时任务（dry-run）：$name，每六小时检查一次"; return }
+if (-not $enabled) {
+    $existing = Get-ScheduledTask -TaskName $name -ErrorAction SilentlyContinue
+    if ($existing) { Unregister-ScheduledTask -TaskName $name -Confirm:$false }
+    return
+}
+$executable = (Get-Process -Id $PID).Path
+$script = Join-Path $ProjectRoot 'scripts/Update-Infrastructure.ps1'
+$action = New-ScheduledTaskAction -Execute $executable -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$script`" -ProjectRoot `"$ProjectRoot`""
+$triggers = @('00:17', '06:17', '12:17', '18:17') | ForEach-Object { New-ScheduledTaskTrigger -Daily -At $_ }
+# Docker Desktop belongs to the logged-in user, not the SYSTEM account.
+$user = [System.Environment]::UserName
+if ($env:USERDOMAIN) { $user = "$env:USERDOMAIN\$user" }
+$principal = New-ScheduledTaskPrincipal -UserId $user -LogonType Interactive -RunLevel Highest
+$settings = New-ScheduledTaskSettingsSet -StartWhenAvailable -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Hours 2)
+Register-ScheduledTask -TaskName $name -Action $action -Trigger $triggers -Principal $principal -Settings $settings -Force | Out-Null
+Write-Host '已配置 Watchtower 和 updater 定时任务：每六小时检查一次（Docker Desktop 可用时执行）'

scripts/Test-InfrastructureUpdateTask.ps1

@@ -0,0 +1,31 @@
+# Test scheduler registration without changing the host's real scheduled tasks.
+$ErrorActionPreference = 'Stop'
+$started = Get-Date
+$root = Join-Path ([System.IO.Path]::GetTempPath()) ('easyai-task-' + [guid]::NewGuid().ToString('N'))
+[System.IO.Directory]::CreateDirectory((Join-Path $root 'scripts')) | Out-Null
+try {
+    Copy-Item (Join-Path $PSScriptRoot 'Initialize-SecurityEnv.ps1') (Join-Path $root 'scripts')
+    Copy-Item (Join-Path $PSScriptRoot 'Install-InfrastructureUpdateTask.ps1') (Join-Path $root 'scripts')
+    $global:infraTaskTestTasks = @{ 'Unrelated-Backup' = @{ keep = $true } }
+    function New-ScheduledTaskAction { param($Execute, $Argument); return @{Execute=$Execute; Argument=$Argument} }
+    function New-ScheduledTaskTrigger { param([switch]$Daily, $At); return @{Daily=$Daily; At=$At} }
+    function New-ScheduledTaskPrincipal { param($UserId, $LogonType, $RunLevel); return @{User=$UserId; Logon=$LogonType; Level=$RunLevel} }
+    function New-ScheduledTaskSettingsSet { param([switch]$StartWhenAvailable, $MultipleInstances, $ExecutionTimeLimit); return @{Concurrent=$MultipleInstances; Available=$StartWhenAvailable} }
+    function Register-ScheduledTask { param($TaskName, $Action, $Trigger, $Principal, $Settings, [switch]$Force); $global:infraTaskTestTasks[$TaskName]=@{Action=$Action; Trigger=$Trigger; Principal=$Principal; Settings=$Settings}; return @{} }
+    function Get-ScheduledTask { param($TaskName, $ErrorAction); return $global:infraTaskTestTasks[$TaskName] }
+    function Unregister-ScheduledTask { param($TaskName, $Confirm); $global:infraTaskTestTasks.Remove($TaskName) }
+    $envFile = Join-Path $root '.env'
+    [System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true')
+    & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
+    & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
+    if ($global:infraTaskTestTasks.Count -ne 2) { throw 'Repeated install duplicated tasks or deleted unrelated tasks' }
+    $key = @($global:infraTaskTestTasks.Keys | Where-Object { $_ -like 'EasyAI-Watchtower-*' })[0]
+    $task = $global:infraTaskTestTasks[$key]
+    if ($task.Trigger.Count -ne 4 -or $task.Settings.Concurrent -ne 'IgnoreNew') { throw 'Schedule or concurrency policy changed' }
+    if ($task.Principal.Logon -ne 'Interactive') { throw 'Task would not use the Docker Desktop user' }
+    if ($task.Action.Argument -notlike '*Update-Infrastructure.ps1*' -or $task.Action.Argument -notlike '*-NonInteractive*') { throw 'Incorrect task command' }
+    [System.IO.File]::WriteAllText($envFile, 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false')
+    & (Join-Path $root 'scripts/Install-InfrastructureUpdateTask.ps1') -ProjectRoot $root
+    if ($global:infraTaskTestTasks.Count -ne 1 -or -not $global:infraTaskTestTasks.ContainsKey('Unrelated-Backup')) { throw 'Disable removed an unrelated task' }
+    Write-Host "PowerShell scheduler contract passed in $([math]::Round(((Get-Date)-$started).TotalSeconds,2)) seconds"
+} finally { Remove-Item -Recurse -Force $root; Remove-Variable infraTaskTestTasks -Scope Global -ErrorAction SilentlyContinue }

scripts/Update-Infrastructure.ps1

@@ -0,0 +1,74 @@
+param([string]$ProjectRoot = (Split-Path $PSScriptRoot -Parent))
+$ErrorActionPreference = 'Stop'
+Set-Location $ProjectRoot
+. (Join-Path $ProjectRoot 'scripts/Initialize-SecurityEnv.ps1')
+$content = [System.IO.File]::ReadAllText((Join-Path $ProjectRoot '.env'))
+if ((Get-EnvValue $content 'INFRASTRUCTURE_AUTO_UPDATE_ENABLED') -eq 'false') { return }
+$state = Join-Path $ProjectRoot 'data/infrastructure-update'
+[System.IO.Directory]::CreateDirectory($state) | Out-Null
+try { $lock = [System.IO.File]::Open((Join-Path $state 'task.lock'), 'OpenOrCreate', 'ReadWrite', 'None') }
+catch [System.IO.IOException] { return }
+$log = Join-Path $state 'last-run.log'
+$previousConfig = $env:DOCKER_CONFIG
+try {
+    [System.IO.File]::WriteAllText($log, "$(Get-Date -Format 'yyyy-MM-dd HH:mm:ss') 检查 Watchtower 和 updater 更新`n")
+    $configFile = Get-EnvValue $content 'DOCKER_CONFIG_FILE'
+    if (-not [string]::IsNullOrWhiteSpace($configFile) -and (Test-Path $configFile -PathType Leaf)) {
+        $env:DOCKER_CONFIG = Split-Path $configFile -Parent
+    }
+    & docker compose version *> $null
+    $v2 = $LASTEXITCODE -eq 0
+    function Invoke-InfrastructureCompose {
+        param([string[]]$ComposeArguments)
+        if ($v2) { $output = & docker compose @ComposeArguments 2>&1 }
+        else { $output = & docker-compose @ComposeArguments 2>&1 }
+        $code = $LASTEXITCODE
+        $output | Out-File $log -Append -Encoding utf8
+        if ($code -ne 0) { throw '组件更新命令未成功，现有容器状态需查看日志' }
+    }
+    if ($v2) { $available = @(& docker compose config --services) }
+    else { $available = @(& docker-compose config --services) }
+    if ($LASTEXITCODE -ne 0 -or $available -notcontains 'watchtower') { throw '无法读取 Watchtower 部署配置' }
+    $failed = @()
+    foreach ($service in @('watchtower', 'easyai-updater')) {
+        # Do not enable updater in profile-disabled legacy deployments.
+        if ($available -notcontains $service) {
+            "$service 未启用，跳过维护" | Out-File $log -Append -Encoding utf8
+            continue
+        }
+        try {
+            "检查 $service" | Out-File $log -Append -Encoding utf8
+            Invoke-InfrastructureCompose -ComposeArguments @('pull', $service)
+            Invoke-InfrastructureCompose -ComposeArguments @('up', '-d', '--no-deps', $service)
+            $healthy = $false
+            for ($attempt=0; $attempt -lt 12; $attempt++) {
+                if ($v2) { $containerId = & docker compose ps -q $service }
+                else { $containerId = & docker-compose ps -q $service }
+                if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" }
+                if ($containerId) {
+                    $status = & docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' $containerId
+                    if ($LASTEXITCODE -ne 0) { throw "无法读取 $service 状态" }
+                    if ($status -eq 'true healthy') { $healthy = $true; break }
+                    if ($status -eq 'true none') {
+                        if ($service -eq 'watchtower') {
+                            & docker exec $containerId /watchtower --health-check *> $null
+                        } else {
+                            & docker exec $containerId node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))" *> $null
+                        }
+                        if ($LASTEXITCODE -eq 0) { $healthy = $true; break }
+                    }
+                }
+                Start-Sleep -Seconds 5
+            }
+            if (-not $healthy) { throw "$service 暂未恢复可用" }
+            "$service 已恢复可用" | Out-File $log -Append -Encoding utf8
+        } catch {
+            "$service 本次维护未成功，下次定时任务将继续检查：$($_.Exception.Message)" | Out-File $log -Append -Encoding utf8
+            $failed += $service
+        }
+    }
+    if ($failed.Count -gt 0) { throw "更新组件维护未全部成功：$($failed -join ', ')" }
+} finally {
+    $env:DOCKER_CONFIG = $previousConfig
+    $lock.Dispose()
+}

scripts/init-infrastructure-update-env.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+. "$PROJECT_ROOT/scripts/init-security-env.sh"
+[ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
+image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)"
+if [ -z "$image" ] || [[ "$image" =~ ^registry\.cn-shanghai\.aliyuncs\.com/comfy-ai/watchtower-aliyun:[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest
+fi
+# Scope changes require the selected containers to receive matching labels.
+# A targeted update must not strand unchanged containers outside the new scope.
+image="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_IMAGE)"
+target="$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_UPDATE_TARGET)"
+if [ -z "${UPDATE_SERVICES:-}" ] && [ -z "$(read_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE)" ] && [ "$image" = registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest ]; then
+    case "$target" in
+        ''|'easyai-updater'|'easyai-server easyai-wsgateway easyai-web')
+            write_env_value "$PROJECT_ROOT/.env" WATCHTOWER_SCOPE easyai
+            ;;
+    esac
+fi

scripts/install-infrastructure-update-task.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -euo pipefail
+PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)"
+. "$PROJECT_ROOT/scripts/init-security-env.sh"
+# A path is shell-quoted and percent-escaped before embedding it in crontab.
+[[ "$PROJECT_ROOT" != *$'\n'* ]] || { echo '部署目录不能包含换行'; exit 1; }
+task_id="$(printf '%s' "$PROJECT_ROOT" | cksum | awk '{print $1}')"
+marker="# easyai-infrastructure-update-$task_id"
+job="$(printf '%q' "$PROJECT_ROOT/scripts/update-infrastructure.sh")"
+job="${job//%/\\%}"
+entry="17 */6 * * * PATH=/usr/local/bin:/opt/homebrew/bin:/usr/sbin:/usr/bin:/sbin:/bin /bin/bash $job >/dev/null 2>&1 $marker"
+if [ "${2:-}" = --dry-run ]; then
+    if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then echo "更新组件定时任务（dry-run）：已关闭"; exit 0; fi
+    printf 'Watchtower 和 updater 定时任务（dry-run）：%s\n' "$entry"
+    exit 0
+fi
+cron() {
+    if [ "$(id -u)" -eq 0 ]; then crontab "$@"; else sudo crontab "$@"; fi
+}
+if ! command -v crontab >/dev/null; then
+    [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
+    if command -v apt-get >/dev/null; then
+        sudo apt-get update -qq
+        sudo apt-get install -y cron
+    else
+        echo '缺少 crontab，无法安装更新组件定时任务' >&2
+        exit 1
+    fi
+fi
+if [ "$(uname -s)" = Linux ]; then
+    if [ -d /run/systemd/system ]; then
+        sudo systemctl enable --now cron
+    elif command -v service >/dev/null; then
+        sudo service cron start
+    fi
+fi
+error_file="$(mktemp)"
+trap 'rm -f "$error_file"' EXIT
+if ! current="$(LC_ALL=C cron -l 2>"$error_file")"; then
+    if ! grep -qi 'no crontab for' "$error_file"; then
+        echo '无法读取现有定时任务，未修改任务配置' >&2
+        exit 1
+    fi
+    current=""
+fi
+# Keep all unrelated tasks and replace this deployment's single entry.
+next="$(printf '%s\n' "$current" | awk -v marker="$marker" 'substr($0, length($0)-length(marker)+1) != marker')"
+if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ]; then
+    next="${next:+$next$'\n'}$entry"
+fi
+printf '%s\n' "$next" | cron -
+if [ "$(read_env_value "$PROJECT_ROOT/.env" INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" = false ]; then
+    echo '已移除本部署的更新组件定时任务'
+else
+    echo '已配置 Watchtower 和 updater 定时任务：每六小时检查一次'
+fi

scripts/test-deployment-public-url.sh

@@ -20,6 +20,9 @@ mkdir -p "$TMP_DIR/scripts"
 mkdir -p "$TMP_DIR/system-update"
 cp \
     "$REPO_ROOT/scripts/init-security-env.sh" \
+    "$REPO_ROOT/scripts/init-infrastructure-update-env.sh" \
+    "$REPO_ROOT/scripts/install-infrastructure-update-task.sh" \
+    "$REPO_ROOT/scripts/update-infrastructure.sh" \
     "$REPO_ROOT/scripts/init-security-origin.sh" \
     "$REPO_ROOT/scripts/init-public-api-base-url.sh" \
     "$REPO_ROOT/scripts/init-server-http-bind-ip.sh" \
@@ -186,6 +189,8 @@ assert str(server_env.get("EASYAI_DEPLOYMENT_SCHEMA_VERSION")) == "2"
 assert str(updater_env.get("UPDATE_HEALTH_TIMEOUT_MS")) == "300000"
 assert updater_env.get("DOCKER_CONFIG_FILE") == "/run/secrets/docker-config.json"
 assert updater_env.get("UPDATE_MANIFEST_FILE") == "/etc/easyai-updater/release-manifest.json"
+assert updater_env.get("UPDATE_MANIFEST_SOURCE") == "remote"
+assert updater_env.get("UPDATE_MANIFEST_URL") == ""
 assert updater.get("labels", {}).get("com.centurylinklabs.watchtower.enable") == "true"
 assert watchtower.get("environment", {}).get("WATCHTOWER_POLL_INTERVAL") == "86400"
 assert watchtower.get("environment", {}).get("WATCHTOWER_CLEANUP") == "true"

scripts/test-infrastructure-auto-update.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Exercise real entry scripts in temporary projects; never edits host schedulers."""
+from pathlib import Path
+import json,os,subprocess,tempfile,time,shutil
+REPO=Path(__file__).resolve().parents[1]
+started=time.monotonic()
+with tempfile.TemporaryDirectory(prefix='easyai infrastructure % ') as tmp:
+ root=Path(tmp); project=root/'project space';(project/'scripts').mkdir(parents=True)
+ for name in ['init-security-env.sh','Initialize-SecurityEnv.ps1','init-infrastructure-update-env.sh','Initialize-InfrastructureUpdateEnv.ps1','install-infrastructure-update-task.sh','Install-InfrastructureUpdateTask.ps1','update-infrastructure.sh','Update-Infrastructure.ps1']:
+  shutil.copy2(REPO/'scripts'/name,project/'scripts'/name)
+ bins=root/'bin';bins.mkdir();calls=root/'calls.jsonl';cron=root/'crontab';cron.write_text('5 2 * * * existing-backup\n# existing comment\n')
+ environment={**os.environ,'PATH':str(bins)+os.pathsep+os.environ['PATH'],'INFRA_TEST_CALLS':str(calls),'INFRA_TEST_CRON':str(cron)}
+ def stub(name,body):
+  p=bins/name;p.write_text(body);p.chmod(0o755)
+ stub('sudo','#!/bin/sh\nexec "$@"\n')
+ stub('service','#!/bin/sh\nexit 0\n')
+ stub('systemctl','#!/bin/sh\nexit 0\n')
+ stub('crontab','''#!/usr/bin/env python3
+import sys,os
+from pathlib import Path
+p=Path(os.environ['INFRA_TEST_CRON'])
+if os.environ.get('INFRA_TEST_CRON_ERROR'):
+ print('permission denied',file=sys.stderr);sys.exit(1)
+if sys.argv[1]=='-l':print(p.read_text(),end='')
+else:p.write_text(sys.stdin.read())
+''')
+ stub('docker','''#!/usr/bin/env python3
+import sys,os,json,time
+from pathlib import Path
+args=sys.argv[1:]
+with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(args)+'\\n')
+if args[:1]==['--config']:args=args[2:]
+if args==['compose','version']:sys.exit(1 if os.environ.get('INFRA_TEST_V1') else 0)
+if args[:3]==['compose','ps','-q']:print(args[3]+'-test-id')
+if args[:1]==['inspect']:print('true none' if os.environ.get('INFRA_TEST_NO_HEALTH') else 'true healthy')
+if args==['compose','config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
+if args[:2]==['compose','pull']:
+ time.sleep(float(os.environ.get('INFRA_TEST_SLEEP','0')))
+ if os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',args[2]]:sys.exit(1)
+''')
+ stub('docker-compose','''#!/usr/bin/env python3
+import sys,os,json
+from pathlib import Path
+with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(['v1']+sys.argv[1:])+'\\n')
+if sys.argv[1:3]==['ps','-q']:print(sys.argv[3]+'-test-id')
+if sys.argv[1:]==['config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
+if sys.argv[1:2]==['pull'] and os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',sys.argv[2]]:sys.exit(1)
+''')
+ def run(args,extra=None):
+  return subprocess.run(args,cwd=project,env={**environment,**(extra or {})},capture_output=True,text=True)
+ def task():return ['bash',str(project/'scripts/update-infrastructure.sh')]
+ def recorded():return [json.loads(line) for line in calls.read_text().splitlines()]
+ for shell in ['bash','pwsh']:
+  for image,enabled,expected in [('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','true','latest'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.13.1','true','latest'),('customer.invalid/watcher:1.22.3','true','1.22.3'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','false','1.22.3')]:
+   (project/'.env').write_text(f'WATCHTOWER_IMAGE={image}\nINFRASTRUCTURE_AUTO_UPDATE_ENABLED={enabled}\nKEEP=original\n')
+   args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
+   p=run(args);assert p.returncode==0,p.stderr
+   content=(project/'.env').read_text();assert f'WATCHTOWER_IMAGE={image.rsplit(":",1)[0]}:{expected}' in content,content
+   assert 'KEEP=original' in content
+   assert run(args).returncode==0 and (project/'.env').read_text()==content
+ for shell in ['bash','pwsh']:
+  (project/'.env').write_text('WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest\nWATCHTOWER_UPDATE_TARGET=easyai-updater\n')
+  args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
+  assert run(args,{'UPDATE_SERVICES':'easyai-server'}).returncode==0
+  assert 'WATCHTOWER_SCOPE=' not in (project/'.env').read_text()
+  assert run(args,{'UPDATE_SERVICES':''}).returncode==0
+  assert 'WATCHTOWER_SCOPE=easyai' in (project/'.env').read_text()
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
+ install=['bash',str(project/'scripts/install-infrastructure-update-task.sh'),str(project)]
+ assert run(install).returncode==0
+ first=cron.read_text();assert 'existing-backup' in first and first.count('# easyai-infrastructure-update-')==1
+ assert '\\%' in first and 'project\\ space' in first,first
+ assert run(install).returncode==0 and cron.read_text()==first
+ assert run(install,{'INFRA_TEST_CRON_ERROR':'1'}).returncode!=0 and cron.read_text()==first
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false\n')
+ assert run(install).returncode==0 and 'easyai-infrastructure-update' not in cron.read_text() and 'existing-backup' in cron.read_text()
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
+ calls.write_text('');assert run(task()).returncode==0
+ assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded()
+ assert ['compose','pull','easyai-updater'] in recorded() and ['compose','up','-d','--no-deps','easyai-updater'] in recorded()
+ assert not any('mongo' in args for args in recorded())
+ calls.write_text('');assert run(task(),{'INFRA_TEST_PULL_FAIL':'1'}).returncode!=0
+ assert not any('up' in args for args in recorded());assert not (project/'data/infrastructure-update/lock').exists()
+ calls.write_text('');p=subprocess.Popen(task(),cwd=project,env={**environment,'INFRA_TEST_SLEEP':'1'},stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+ for _ in range(100):
+  if (project/'data/infrastructure-update/lock/pid').exists():break
+  time.sleep(.01)
+ assert run(task()).returncode==0
+ assert p.wait(timeout=5)==0 and sum('pull' in args for args in recorded())==2
+ calls.write_text('');assert run(task(),{'INFRA_TEST_V1':'1'}).returncode==0
+ assert ['v1','up','-d','--no-deps','watchtower'] in recorded() and ['v1','up','-d','--no-deps','easyai-updater'] in recorded()
+ calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)])
+ assert p.returncode==0,p.stderr
+ assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded(),recorded()
+ calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)],{'INFRA_TEST_PULL_FAIL':'1'})
+ assert p.returncode!=0 and not any('up' in args for args in recorded())
+ for shell in ['bash','pwsh']:
+  args=task() if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)]
+  for transport in ['', '1']:
+   kind='v1' if transport else 'compose'
+   for failed,other in [('watchtower','easyai-updater'),('easyai-updater','watchtower')]:
+    calls.write_text('');p=run(args,{'INFRA_TEST_PULL_FAIL':failed,'INFRA_TEST_V1':transport})
+    assert p.returncode!=0,(shell,p.stderr)
+    assert [kind,'pull',failed] in recorded() and [kind,'pull',other] in recorded()
+    assert [kind,'up','-d','--no-deps',failed] not in recorded()
+    assert [kind,'up','-d','--no-deps',other] in recorded(),(shell,recorded())
+   calls.write_text('');p=run(args,{'INFRA_TEST_LEGACY':'1','INFRA_TEST_V1':transport})
+   assert p.returncode==0 and not any('easyai-updater' in a for a in recorded()),(shell,p.stderr,recorded())
+  calls.write_text('');p=run(args,{'INFRA_TEST_NO_HEALTH':'1'})
+  assert p.returncode==0,(shell,p.stderr)
+  assert any(a[:3]==['exec','easyai-updater-test-id','node'] for a in recorded())
+print(json.dumps({'passed':True,'cases':['Bash/PowerShell default migration and explicit choices','cron registration deduplication and existing task preservation','cron read failure leaves jobs untouched','Bash/PowerShell independent component maintenance and partial failure reporting','legacy profile skips updater','concurrent task excluded','Compose v1 compatibility','only Watchtower and updater updated','service health probes'],'durationSeconds':round(time.monotonic()-started,3)}))

scripts/test-model-conversion-update.py

@@ -34,6 +34,8 @@ def fixture(base):
             shutil.copytree(path, root / name)
         else:
             shutil.copy2(path, root / name)
+    (root / 'scripts/install-infrastructure-update-task.sh').write_text('#!/bin/sh\nexit 0\n')
+    (root / 'scripts/Install-InfrastructureUpdateTask.ps1').write_text('# Scheduler transport is verified separately.\n')
     (root / 'data').mkdir()
     (root / '.env').write_text('SYSTEM_UPDATE_MODE=legacy_watchtower\nNUXT_PUBLIC_BASE_APIURL=http://127.0.0.1:3001\nCONFIG_JWT_SECRET=existing-jwt-secret-that-is-long-enough\nWS_AUTH_WS_TICKET_SECRET=existing-ticket-secret-that-is-long-enough\n')
     (root / '.gitignore').write_text('.env\n*.log\n')

scripts/test-update-env-migrations.sh

@@ -26,7 +26,7 @@ grep -qx 'COMPOSE_PROFILES=customer-profile' "$LEGACY_ENV_FILE"
 grep -qx 'WATCHTOWER_CLEANUP=' "$LEGACY_ENV_FILE"
 grep -qx 'WATCHTOWER_HTTP_API_PERIODIC_POLLS=false' "$LEGACY_ENV_FILE"
 grep -qx 'WATCHTOWER_UPDATE_TARGET=easyai-server easyai-wsgateway easyai-web' "$LEGACY_ENV_FILE"
-grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3' "$LEGACY_ENV_FILE"
+grep -qx 'WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest' "$LEGACY_ENV_FILE"
 grep -qx 'MODEL_CONVERSION_WORKER_VERSION=latest' "$LEGACY_ENV_FILE"
 grep -qx 'MODEL_CONVERSION_HUNYUAN_ENABLED=false' "$LEGACY_ENV_FILE"
 cp "$LEGACY_ENV_FILE" "$TMP_DIR/legacy-first-run.env"

scripts/update-env.d/020-watchtower-progress.env

@@ -1,2 +1,2 @@
 # 两种更新模式均启用内网接口；安全令牌由安全初始化脚本生成。
-WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3
+WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest

scripts/update-env.d/023-infrastructure-auto-update.env

@@ -0,0 +1,2 @@
+# 宿主机定时维护更新组件，保留客户明确关闭的选择。
+INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true

scripts/update-infrastructure.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+set -euo pipefail
+PROJECT_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+cd "$PROJECT_ROOT"
+. ./scripts/init-security-env.sh
+[ "$(read_env_value .env INFRASTRUCTURE_AUTO_UPDATE_ENABLED)" != false ] || exit 0
+state="$PROJECT_ROOT/data/infrastructure-update"
+mkdir -p "$state"
+lock="$state/lock"
+if ! mkdir "$lock" 2>/dev/null; then
+    pid="$(cat "$lock/pid" 2>/dev/null || true)"
+    if [[ "$pid" =~ ^[0-9]+$ ]] && ! kill -0 "$pid" 2>/dev/null; then
+        rm -f "$lock/pid"
+        rmdir "$lock" 2>/dev/null || exit 0
+        mkdir "$lock" 2>/dev/null || exit 0
+    else
+        exit 0
+    fi
+fi
+printf '%s\n' "$$" > "$lock/pid"
+trap 'rm -f "$lock/pid"; rmdir "$lock" 2>/dev/null || true' EXIT
+exec > "$state/last-run.log" 2>&1
+printf '%s 检查 Watchtower 和 updater 更新\n' "$(date '+%Y-%m-%d %H:%M:%S')"
+registry_config="$(read_env_value .env DOCKER_CONFIG_FILE)"
+docker_options=()
+if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then
+    docker_options=(--config "$(dirname "$registry_config")")
+fi
+if docker "${docker_options[@]}" compose version >/dev/null 2>&1; then
+    compose() { docker "${docker_options[@]}" compose "$@"; }
+else
+    compose() {
+        if [ -n "$registry_config" ] && [ -f "$registry_config" ]; then
+            DOCKER_CONFIG="$(dirname "$registry_config")" docker-compose "$@"
+        else
+            docker-compose "$@"
+        fi
+    }
+fi
+available="$(compose config --services)"
+printf '%s\n' "$available" | grep -qx watchtower || { echo '未配置 Watchtower'; exit 1; }
+maintain_service() {
+    local service="$1" container_id status attempt
+    echo "检查 $service"
+    # A failed service must not block the other update channel.
+    compose pull "$service" || return 1
+    compose up -d --no-deps "$service" || return 1
+    for ((attempt=0; attempt<12; attempt++)); do
+        container_id="$(compose ps -q "$service")" || return 1
+        if [ -n "$container_id" ]; then
+            status="$(docker inspect -f '{{.State.Running}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id")" || return 1
+            if [ "$status" = 'true healthy' ]; then
+                echo "$service 已恢复可用"
+                return 0
+            fi
+            if [ "$status" = 'true none' ]; then
+                if [ "$service" = watchtower ]; then
+                    if docker exec "$container_id" /watchtower --health-check; then return 0; fi
+                else
+                    if docker exec "$container_id" node -e "require('http').get('http://127.0.0.1:3015/health',r=>{r.resume();process.exit(r.statusCode===200?0:1)}).on('error',()=>process.exit(1))"; then return 0; fi
+                fi
+            fi
+        fi
+        sleep 5
+    done
+    echo "$service 暂未恢复可用" >&2
+    return 1
+}
+failed=0
+for service in watchtower easyai-updater; do
+    # config --services respects active profiles: legacy deployments stay legacy.
+    if ! printf '%s\n' "$available" | grep -qx "$service"; then
+        echo "$service 未启用，跳过维护"
+        continue
+    fi
+    if ! maintain_service "$service"; then
+        echo "$service 本次维护未成功，下次定时任务将继续检查" >&2
+        failed=1
+    fi
+done
+exit "$failed"