forked from wangbo/easyai
113 lines
8.7 KiB
Python
113 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Exercise real entry scripts in temporary projects; never edits host schedulers."""
|
|
from pathlib import Path
|
|
import json,os,subprocess,tempfile,time,shutil
|
|
REPO=Path(__file__).resolve().parents[1]
|
|
started=time.monotonic()
|
|
with tempfile.TemporaryDirectory(prefix='easyai infrastructure % ') as tmp:
|
|
root=Path(tmp); project=root/'project space';(project/'scripts').mkdir(parents=True)
|
|
for name in ['init-security-env.sh','Initialize-SecurityEnv.ps1','init-infrastructure-update-env.sh','Initialize-InfrastructureUpdateEnv.ps1','install-infrastructure-update-task.sh','Install-InfrastructureUpdateTask.ps1','update-infrastructure.sh','Update-Infrastructure.ps1']:
|
|
shutil.copy2(REPO/'scripts'/name,project/'scripts'/name)
|
|
bins=root/'bin';bins.mkdir();calls=root/'calls.jsonl';cron=root/'crontab';cron.write_text('5 2 * * * existing-backup\n# existing comment\n')
|
|
environment={**os.environ,'PATH':str(bins)+os.pathsep+os.environ['PATH'],'INFRA_TEST_CALLS':str(calls),'INFRA_TEST_CRON':str(cron)}
|
|
def stub(name,body):
|
|
p=bins/name;p.write_text(body);p.chmod(0o755)
|
|
stub('sudo','#!/bin/sh\nexec "$@"\n')
|
|
stub('service','#!/bin/sh\nexit 0\n')
|
|
stub('systemctl','#!/bin/sh\nexit 0\n')
|
|
stub('crontab','''#!/usr/bin/env python3
|
|
import sys,os
|
|
from pathlib import Path
|
|
p=Path(os.environ['INFRA_TEST_CRON'])
|
|
if os.environ.get('INFRA_TEST_CRON_ERROR'):
|
|
print('permission denied',file=sys.stderr);sys.exit(1)
|
|
if sys.argv[1]=='-l':print(p.read_text(),end='')
|
|
else:p.write_text(sys.stdin.read())
|
|
''')
|
|
stub('docker','''#!/usr/bin/env python3
|
|
import sys,os,json,time
|
|
from pathlib import Path
|
|
args=sys.argv[1:]
|
|
with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(args)+'\\n')
|
|
if args[:1]==['--config']:args=args[2:]
|
|
if args==['compose','version']:sys.exit(1 if os.environ.get('INFRA_TEST_V1') else 0)
|
|
if args[:3]==['compose','ps','-q']:print(args[3]+'-test-id')
|
|
if args[:1]==['inspect']:print('true none' if os.environ.get('INFRA_TEST_NO_HEALTH') else 'true healthy')
|
|
if args==['compose','config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
|
|
if args[:2]==['compose','pull']:
|
|
time.sleep(float(os.environ.get('INFRA_TEST_SLEEP','0')))
|
|
if os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',args[2]]:sys.exit(1)
|
|
''')
|
|
stub('docker-compose','''#!/usr/bin/env python3
|
|
import sys,os,json
|
|
from pathlib import Path
|
|
with Path(os.environ['INFRA_TEST_CALLS']).open('a') as f:f.write(json.dumps(['v1']+sys.argv[1:])+'\\n')
|
|
if sys.argv[1:3]==['ps','-q']:print(sys.argv[3]+'-test-id')
|
|
if sys.argv[1:]==['config','--services']:print('watchtower\\nmongo' if os.environ.get('INFRA_TEST_LEGACY') else 'watchtower\\neasyai-updater\\nmongo')
|
|
if sys.argv[1:2]==['pull'] and os.environ.get('INFRA_TEST_PULL_FAIL') in ['1',sys.argv[2]]:sys.exit(1)
|
|
''')
|
|
def run(args,extra=None):
|
|
return subprocess.run(args,cwd=project,env={**environment,**(extra or {})},capture_output=True,text=True)
|
|
def task():return ['bash',str(project/'scripts/update-infrastructure.sh')]
|
|
def recorded():return [json.loads(line) for line in calls.read_text().splitlines()]
|
|
for shell in ['bash','pwsh']:
|
|
for image,enabled,expected in [('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','true','latest'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.13.1','true','latest'),('customer.invalid/watcher:1.22.3','true','1.22.3'),('registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:1.22.3','false','1.22.3')]:
|
|
(project/'.env').write_text(f'WATCHTOWER_IMAGE={image}\nINFRASTRUCTURE_AUTO_UPDATE_ENABLED={enabled}\nKEEP=original\n')
|
|
args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
|
|
p=run(args);assert p.returncode==0,p.stderr
|
|
content=(project/'.env').read_text();assert f'WATCHTOWER_IMAGE={image.rsplit(":",1)[0]}:{expected}' in content,content
|
|
assert 'KEEP=original' in content
|
|
assert run(args).returncode==0 and (project/'.env').read_text()==content
|
|
for shell in ['bash','pwsh']:
|
|
(project/'.env').write_text('WATCHTOWER_IMAGE=registry.cn-shanghai.aliyuncs.com/comfy-ai/watchtower-aliyun:latest\nWATCHTOWER_UPDATE_TARGET=easyai-updater\n')
|
|
args=['bash',str(project/'scripts/init-infrastructure-update-env.sh'),str(project)] if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Initialize-InfrastructureUpdateEnv.ps1'),'-ProjectRoot',str(project)]
|
|
assert run(args,{'UPDATE_SERVICES':'easyai-server'}).returncode==0
|
|
assert 'WATCHTOWER_SCOPE=' not in (project/'.env').read_text()
|
|
assert run(args,{'UPDATE_SERVICES':''}).returncode==0
|
|
assert 'WATCHTOWER_SCOPE=easyai' in (project/'.env').read_text()
|
|
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
|
|
install=['bash',str(project/'scripts/install-infrastructure-update-task.sh'),str(project)]
|
|
assert run(install).returncode==0
|
|
first=cron.read_text();assert 'existing-backup' in first and first.count('# easyai-infrastructure-update-')==1
|
|
assert '\\%' in first and 'project\\ space' in first,first
|
|
assert run(install).returncode==0 and cron.read_text()==first
|
|
assert run(install,{'INFRA_TEST_CRON_ERROR':'1'}).returncode!=0 and cron.read_text()==first
|
|
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=false\n')
|
|
assert run(install).returncode==0 and 'easyai-infrastructure-update' not in cron.read_text() and 'existing-backup' in cron.read_text()
|
|
(project/'.env').write_text('INFRASTRUCTURE_AUTO_UPDATE_ENABLED=true\n')
|
|
calls.write_text('');assert run(task()).returncode==0
|
|
assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded()
|
|
assert ['compose','pull','easyai-updater'] in recorded() and ['compose','up','-d','--no-deps','easyai-updater'] in recorded()
|
|
assert not any('mongo' in args for args in recorded())
|
|
calls.write_text('');assert run(task(),{'INFRA_TEST_PULL_FAIL':'1'}).returncode!=0
|
|
assert not any('up' in args for args in recorded());assert not (project/'data/infrastructure-update/lock').exists()
|
|
calls.write_text('');p=subprocess.Popen(task(),cwd=project,env={**environment,'INFRA_TEST_SLEEP':'1'},stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
|
for _ in range(100):
|
|
if (project/'data/infrastructure-update/lock/pid').exists():break
|
|
time.sleep(.01)
|
|
assert run(task()).returncode==0
|
|
assert p.wait(timeout=5)==0 and sum('pull' in args for args in recorded())==2
|
|
calls.write_text('');assert run(task(),{'INFRA_TEST_V1':'1'}).returncode==0
|
|
assert ['v1','up','-d','--no-deps','watchtower'] in recorded() and ['v1','up','-d','--no-deps','easyai-updater'] in recorded()
|
|
calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)])
|
|
assert p.returncode==0,p.stderr
|
|
assert ['compose','pull','watchtower'] in recorded() and ['compose','up','-d','--no-deps','watchtower'] in recorded(),recorded()
|
|
calls.write_text('');p=run(['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)],{'INFRA_TEST_PULL_FAIL':'1'})
|
|
assert p.returncode!=0 and not any('up' in args for args in recorded())
|
|
for shell in ['bash','pwsh']:
|
|
args=task() if shell=='bash' else ['pwsh','-NoProfile','-File',str(project/'scripts/Update-Infrastructure.ps1'),'-ProjectRoot',str(project)]
|
|
for transport in ['', '1']:
|
|
kind='v1' if transport else 'compose'
|
|
for failed,other in [('watchtower','easyai-updater'),('easyai-updater','watchtower')]:
|
|
calls.write_text('');p=run(args,{'INFRA_TEST_PULL_FAIL':failed,'INFRA_TEST_V1':transport})
|
|
assert p.returncode!=0,(shell,p.stderr)
|
|
assert [kind,'pull',failed] in recorded() and [kind,'pull',other] in recorded()
|
|
assert [kind,'up','-d','--no-deps',failed] not in recorded()
|
|
assert [kind,'up','-d','--no-deps',other] in recorded(),(shell,recorded())
|
|
calls.write_text('');p=run(args,{'INFRA_TEST_LEGACY':'1','INFRA_TEST_V1':transport})
|
|
assert p.returncode==0 and not any('easyai-updater' in a for a in recorded()),(shell,p.stderr,recorded())
|
|
calls.write_text('');p=run(args,{'INFRA_TEST_NO_HEALTH':'1'})
|
|
assert p.returncode==0,(shell,p.stderr)
|
|
assert any(a[:3]==['exec','easyai-updater-test-id','node'] for a in recorded())
|
|
print(json.dumps({'passed':True,'cases':['Bash/PowerShell default migration and explicit choices','cron registration deduplication and existing task preservation','cron read failure leaves jobs untouched','Bash/PowerShell independent component maintenance and partial failure reporting','legacy profile skips updater','concurrent task excluded','Compose v1 compatibility','only Watchtower and updater updated','service health probes'],'durationSeconds':round(time.monotonic()-started,3)}))
|