feat(deploy): 为更新组件增加定时维护与双保险

scripts/test-infrastructure-auto-update.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Exercise real entry scripts in temporary projects; never edits host schedulers."""
+from pathlib import Path
+import json,os,subprocess,tempfile,time,shutil
+REPO=Path(__file__).resolve().parents[1]
+started=time.monotonic()
+with tempfile.TemporaryDirectory(prefix='easyai infrastructure % ') as tmp:
+ root=Path(tmp); project=root/'project space';(project/'scripts').mkdir(parents=True)
+ for name in ['init-security-env.sh','Initialize-SecurityEnv.ps1','init-infrastructure-update-env.sh','Initialize-InfrastructureUpdateEnv.ps1','install-infrastructure-update-task.sh','Install-InfrastructureUpdateTask.ps1','update-infrastructure.sh','Update-Infrastructure.ps1']:
+  shutil.copy2(REPO/'scripts'/name,project/'scripts'/name)
+ bins=root/'bin';bins.mkdir();calls=root/'calls.jsonl';cron=root/'crontab';cron.write_text('5 2 * * * existing-backup\n# existing comment\n')
+ environment={**os.environ,'PATH':str(bins)+os.pathsep+os.environ['PATH'],'INFRA_TEST_CALLS':str(calls),'INFRA_TEST_CRON':str(cron)}
+ def stub(name,body):
+  p=bins/name;p.write_text(body);p.chmod(0o755)
+ stub('sudo','#!/bin/sh\nexec "$@"\n')
+ stub('service','#!/bin/sh\nexit 0\n')
+ stub('systemctl','#!/bin/sh\nexit 0\n')
+ stub('crontab','''#!/usr/bin/env python3
+import sys,os
+from pathlib import Path
+p=Path(os.environ['INFRA_TEST_CRON'])
+if os.environ.get('INFRA_TEST_CRON_ERROR'):
+ print('permission denied',file=sys.stderr);sys.exit(1)
+if sys.argv[1]=='-l':print(p.read_text(),end='')
+else:p.write_text(sys.stdin.read())
+''')
+ stub('docker','''#!/usr/bin/env python3
+import sys,os,json,time
+from pathlib import Path
+args=sys.argv[1:]
+with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(args)+'\\n')
+if args[:1]==['--config']:args=args[2:]
+if args==['compose','version']:sys.exit(1 if os.environ.get('INFRA_TEST_V1') else 0)
+if args[:3]==['compose','ps','-q']:print(args[3]+'-test-id')
+if args[:1]==['inspect']:print('true none' if os.environ.get('INFRA_TEST_NO_HEALTH') else 'true healthy')
+if args==['compose','config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
+if args[:2]==['compose','pull']:
+ time.sleep(float(os.environ.get('INFRA_TEST_SLEEP','0')))
+ if os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',args[2]]:sys.exit(1)
+''')
+ stub('docker-compose','''#!/usr/bin/env python3
+import sys,os,json
+from pathlib import Path
+with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(['v1']+sys.argv[1:])+'\\n')
+if sys.argv[1:3]==['ps','-q']:print(sys.argv[3]+'-test-id')
+if sys.argv[1:]==['config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
+if sys.argv[1:2]==['pull'] and os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',sys.argv[2]]:sys.exit(1)
+''')
+ def run(args,extra=None):
+  return subprocess.run(args,cwd=project,env={**environment,**(extra or {})},capture_output=True,text=True)
+ def task():return ['bash',str(project/'scripts/update-infrastructure.sh')]
+ def recorded():return [json.loads(line) for line in calls.read_text().splitlines()]
+ for shell in ['bash','pwsh']:
+  for image,enabled,expected in [('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','true','latest'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.13.1','true','latest'),('customer.invalid/watcher:1.22.3','true','1.22.3'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','false','1.22.3')]:
+   (project/'.env').write_text(f'WATCHTOWER_IMAGE={image}\nINFRASTRUCTURE_AUTO_UPDATE_ENABLED={enabled}\nKEEP=original\n')
+   args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
+   p=run(args);assert p.returncode==0,p.stderr
+   content=(project/'.env').read_text();assert f'WATCHTOWER_IMAGE={image.rsplit(":",1)[0]}:{expected}' in content,content
+   assert 'KEEP=original' in content
+   assert run(args).returncode==0 and (project/'.env').read_text()==content
+ for shell in ['bash','pwsh']:
+  (project/'.env').write_text('WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest\nWATCHTOWER_UPDATE_TARGET=easyai-updater\n')
+  args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
+  assert run(args,{'UPDATE_SERVICES':'easyai-server'}).returncode==0
+  assert 'WATCHTOWER_SCOPE=' not in (project/'.env').read_text()
+  assert run(args,{'UPDATE_SERVICES':''}).returncode==0
+  assert 'WATCHTOWER_SCOPE=easyai' in (project/'.env').read_text()
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
+ install=['bash',str(project/'scripts/install-infrastructure-update-task.sh'),str(project)]
+ assert run(install).returncode==0
+ first=cron.read_text();assert 'existing-backup' in first and first.count('# easyai-infrastructure-update-')==1
+ assert '\\%' in first and 'project\\ space' in first,first
+ assert run(install).returncode==0 and cron.read_text()==first
+ assert run(install,{'INFRA_TEST_CRON_ERROR':'1'}).returncode!=0 and cron.read_text()==first
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false\n')
+ assert run(install).returncode==0 and 'easyai-infrastructure-update' not in cron.read_text() and 'existing-backup' in cron.read_text()
+ (project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
+ calls.write_text('');assert run(task()).returncode==0
+ assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded()
+ assert ['compose','pull','easyai-updater'] in recorded() and ['compose','up','-d','--no-deps','easyai-updater'] in recorded()
+ assert not any('mongo' in args for args in recorded())
+ calls.write_text('');assert run(task(),{'INFRA_TEST_PULL_FAIL':'1'}).returncode!=0
+ assert not any('up' in args for args in recorded());assert not (project/'data/infrastructure-update/lock').exists()
+ calls.write_text('');p=subprocess.Popen(task(),cwd=project,env={**environment,'INFRA_TEST_SLEEP':'1'},stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+ for _ in range(100):
+  if (project/'data/infrastructure-update/lock/pid').exists():break
+  time.sleep(.01)
+ assert run(task()).returncode==0
+ assert p.wait(timeout=5)==0 and sum('pull' in args for args in recorded())==2
+ calls.write_text('');assert run(task(),{'INFRA_TEST_V1':'1'}).returncode==0
+ assert ['v1','up','-d','--no-deps','watchtower'] in recorded() and ['v1','up','-d','--no-deps','easyai-updater'] in recorded()
+ calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)])
+ assert p.returncode==0,p.stderr
+ assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded(),recorded()
+ calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)],{'INFRA_TEST_PULL_FAIL':'1'})
+ assert p.returncode!=0 and not any('up' in args for args in recorded())
+ for shell in ['bash','pwsh']:
+  args=task() if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)]
+  for transport in ['', '1']:
+   kind='v1' if transport else 'compose'
+   for failed,other in [('watchtower','easyai-updater'),('easyai-updater','watchtower')]:
+    calls.write_text('');p=run(args,{'INFRA_TEST_PULL_FAIL':failed,'INFRA_TEST_V1':transport})
+    assert p.returncode!=0,(shell,p.stderr)
+    assert [kind,'pull',failed] in recorded() and [kind,'pull',other] in recorded()
+    assert [kind,'up','-d','--no-deps',failed] not in recorded()
+    assert [kind,'up','-d','--no-deps',other] in recorded(),(shell,recorded())
+   calls.write_text('');p=run(args,{'INFRA_TEST_LEGACY':'1','INFRA_TEST_V1':transport})
+   assert p.returncode==0 and not any('easyai-updater' in a for a in recorded()),(shell,p.stderr,recorded())
+  calls.write_text('');p=run(args,{'INFRA_TEST_NO_HEALTH':'1'})
+  assert p.returncode==0,(shell,p.stderr)
+  assert any(a[:3]==['exec','easyai-updater-test-id','node'] for a in recorded())
+print(json.dumps({'passed':True,'cases':['Bash/PowerShell default migration and explicit choices','cron registration deduplication and existing task preservation','cron read failure leaves jobs untouched','Bash/PowerShell independent component maintenance and partial failure reporting','legacy profile skips updater','concurrent task excluded','Compose v1 compatibility','only Watchtower and updater updated','service health probes'],'durationSeconds':round(time.monotonic()-started,3)}))