#!/usr/bin/env python3 """Download pinned public tools/sources; verify hashes before extraction/use.""" import argparse, hashlib, json, os, shutil, stat, tarfile, urllib.request, zipfile from pathlib import Path, PurePosixPath ROOT=Path(__file__).resolve().parents[1] def sha(p): h=hashlib.sha256() with p.open('rb') as f: for data in iter(lambda:f.read(1024*1024),b''):h.update(data) return h.hexdigest() def fetch(url,path,expected): path.parent.mkdir(parents=True,exist_ok=True) if path.exists(): if sha(path)!=expected:raise RuntimeError('Preserve invalid cached download: '+str(path)) return path temporary=path.with_name(path.name+'.partial') if temporary.exists():raise RuntimeError('Preserve incomplete download: '+str(temporary)) print('DOWNLOAD '+url,flush=True) request=urllib.request.Request(url,headers={'User-Agent':'Coco3Plus-build/1.0'}) with urllib.request.urlopen(request,timeout=120) as response,temporary.open('xb') as out: shutil.copyfileobj(response,out) if sha(temporary)!=expected:raise RuntimeError('Download hash mismatch: '+str(temporary)) temporary.rename(path);return path def safe_member(name): p=PurePosixPath(name) if p.is_absolute() or '..' in p.parts or '\\' in name or ':' in name: raise RuntimeError('Unsafe archive member: '+name) return p def install_simulator(archive,folder,archive_hash): marker='.coco3plus-install.json' if folder.exists(): record=folder/marker if not record.is_file():raise RuntimeError('Incomplete/legacy simulator directory; preserve it and use a fresh --simulator-dir: '+str(folder)) data=json.loads(record.read_text()) if data['archive_sha256']!=archive_hash:raise RuntimeError('Simulator archive identity changed') for name,item in data['files'].items(): path=folder/safe_member(name) if not path.is_file() or path.is_symlink() or path.stat().st_size!=item['bytes'] or sha(path)!=item['sha256']:raise RuntimeError('Simulator installation incomplete/changed: '+name) return partial=folder.with_name(folder.name+'.partial') if partial.exists():raise RuntimeError('Preserve incomplete simulator extraction: '+str(partial)) partial.mkdir(parents=True);files={} with tarfile.open(archive,'r:gz') as tar: for member in tar.getmembers(): rel=safe_member(member.name);target=partial/rel if not(member.isfile() or member.isdir()):raise RuntimeError('Unsupported simulator entry: '+member.name) if member.isdir():target.mkdir(parents=True,exist_ok=True) else: if rel.as_posix() in files:raise RuntimeError('Duplicate simulator entry') target.parent.mkdir(parents=True,exist_ok=True) with tar.extractfile(member) as source,target.open('xb') as out:shutil.copyfileobj(source,out) files[rel.as_posix()]={'bytes':target.stat().st_size,'sha256':sha(target)} for binary in ('iverilog.exe','vvp.exe'): if 'oss-cad-suite/bin/'+binary not in files:raise RuntimeError('Missing simulator executable: '+binary) (partial/marker).write_text(json.dumps(dict(archive_sha256=archive_hash,files=files),indent=2)+'\n') partial.rename(folder) def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--cache',type=Path,default=ROOT/'.cache/downloads') p.add_argument('--simulator-dir',type=Path,default=ROOT/'.tools/oss-cad') p.add_argument('--only',choices=['lwtools','cocoroms','main','simulator'],nargs='+',default=['lwtools','cocoroms','main']) a=p.parse_args();lock=json.loads((ROOT/'tools/toolchain.json').read_text()) if 'lwtools' in a.only: if os.name!='nt':raise RuntimeError('The pinned prebuilt LWTools package is Windows x64. Supply native lwasm to build_roms.py on other systems.') item=lock['lwtools_windows'];archive=fetch(item['url'],a.cache/item['archive'],item['sha256']) folder=ROOT/'.tools/lwtools' if not folder.exists(): folder.mkdir(parents=True) with zipfile.ZipFile(archive) as z: for m in z.infolist(): rel=safe_member(m.filename) if stat.S_ISLNK(m.external_attr>>16):raise RuntimeError('Archive symlink') target=folder/rel if m.is_dir():target.mkdir(parents=True,exist_ok=True) else:target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes(z.read(m)) found=list(folder.rglob('lwasm.exe')) if len(found)!=1 or sha(found[0])!=item['lwasm_sha256']:raise RuntimeError('Pinned LWASM not found or changed') print('LWASM '+str(found[0])) if 'cocoroms' in a.only: item=json.loads((ROOT/'tools/cocoroms.json').read_text()) folder=ROOT/'.deps/cocoroms';folder.mkdir(parents=True,exist_ok=True) for name,value in item['upstreamFileSHA256'].items(): url=f"https://raw.githubusercontent.com/nitros9project/toolshed/{item['sourceRevision']}/cocoroms/{name}" cached=fetch(url,a.cache/('cocoroms-'+item['sourceRevision'])/name,value) target=folder/name if target.exists() and sha(target)!=value:raise RuntimeError('Prepared assembly changed: '+name) if not target.exists():shutil.copyfile(cached,target) print('ToolShed assembly: eight source files verified; no prebuilt ROM input') if 'main' in a.only: item=lock['main_reference'];folder=ROOT/'.deps/main';folder.mkdir(parents=True,exist_ok=True) for name,record in item['files'].items(): cached=fetch(f"https://raw.githubusercontent.com/MiSTer-devel/Main_MiSTer/{item['revision']}/{name}",a.cache/('main-'+item['revision'])/name,record['sha256']) target=folder/name if target.exists() and sha(target)!=record['sha256']:raise RuntimeError('Main reference changed: '+name) if not target.exists():shutil.copyfile(cached,target) (folder/'provenance.json').write_text(json.dumps(item,indent=2)+'\n') print('Main parser reference verified (not installed or modified)') if 'simulator' in a.only: item=lock['oss_cad_windows'];archive=fetch(item['url'],a.cache/item['archive'],item['sha256']) folder=a.simulator_dir.resolve() install_simulator(archive,folder,item['sha256']) print('Simulator '+str(folder/'oss-cad-suite')) if __name__=='__main__':main()