sync: WIN10-I9-64G 2026-10-19
This commit is contained in:
Myfro9
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""merge_rootfs.py -- assemble a release root filesystem tarball from a JSON spec.
|
||||
|
||||
python merge_rootfs.py <spec.json> <out.tar.gz> [--verify]
|
||||
|
||||
Works on Windows or Linux without root: tar members are copied as TarInfo, so
|
||||
owners, modes, symlinks, hard links and device nodes survive exactly (an
|
||||
extract-and-retar on Windows, or as non-root on Linux, would lose them).
|
||||
Later layers override earlier ones by path; the output is deterministic
|
||||
(fixed mtime for added entries, gzip header mtime 0), so the same spec gives
|
||||
the same archive bytes.
|
||||
|
||||
spec.json (paths relative to the spec file unless absolute):
|
||||
{
|
||||
"mtime": 1792022400, # added entries; optional
|
||||
"layers": [ "rootfs.tar.gz", "overlay.tar.gz" ], # applied in order
|
||||
"remove": [ "etc/dropbear/dropbear_rsa_host_key" ], # paths (and their subtrees)
|
||||
"files": [ {"path": "usr/bin/app", "src": "bin/app", "mode": "0755"} ],
|
||||
"links": [ {"path": "etc/app/current.cfg", "target": "/etc/app/a.cfg"} ],
|
||||
"dirs": [ "aw/dpd_curve" ],
|
||||
"expect": { "usr/bin/app": "<sha256 or a prefix>" } # checked with --verify
|
||||
}
|
||||
"""
|
||||
import gzip, hashlib, io, json, os, sys, tarfile
|
||||
|
||||
|
||||
def norm(n):
|
||||
while n.startswith("./"):
|
||||
n = n[2:]
|
||||
n = n.strip("/")
|
||||
return "./" + n if n not in ("", ".") else "./"
|
||||
|
||||
|
||||
class Tree:
|
||||
def __init__(self):
|
||||
self.m, self.order = {}, []
|
||||
|
||||
def put(self, ti, data):
|
||||
ti.name = norm(ti.name)
|
||||
if ti.islnk():
|
||||
ti.linkname = norm(ti.linkname)
|
||||
if ti.name not in self.m:
|
||||
self.order.append(ti.name)
|
||||
self.m[ti.name] = (ti, data)
|
||||
|
||||
def remove(self, p):
|
||||
p = norm(p)
|
||||
for n in list(self.m):
|
||||
if n == p or n.startswith(p + "/"):
|
||||
del self.m[n]
|
||||
self.order = [n for n in self.order if n in self.m]
|
||||
|
||||
|
||||
def info(name, kind, mode, mtime):
|
||||
ti = tarfile.TarInfo(name); ti.type = kind; ti.mode = mode; ti.mtime = mtime
|
||||
ti.uid = ti.gid = 0; ti.uname = ti.gname = "root"
|
||||
return ti
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
spec_path, out = argv[1], argv[2]
|
||||
base = os.path.dirname(os.path.abspath(spec_path))
|
||||
rel = lambda p: p if os.path.isabs(p) else os.path.join(base, p)
|
||||
spec = json.load(open(spec_path))
|
||||
mt = int(spec.get("mtime", 1700000000))
|
||||
t = Tree()
|
||||
for layer in spec.get("layers", []):
|
||||
with tarfile.open(rel(layer), "r:*") as tf:
|
||||
for ti in tf.getmembers():
|
||||
t.put(ti, tf.extractfile(ti).read() if ti.isfile() else None)
|
||||
for p in spec.get("remove", []):
|
||||
t.remove(p)
|
||||
for f in spec.get("files", []):
|
||||
d = open(rel(f["src"]), "rb").read()
|
||||
ti = info(f["path"], tarfile.REGTYPE, int(f.get("mode", "0644"), 8), mt); ti.size = len(d)
|
||||
t.put(ti, d)
|
||||
for l in spec.get("links", []):
|
||||
ti = info(l["path"], tarfile.SYMTYPE, 0o777, mt); ti.linkname = l["target"]
|
||||
t.put(ti, None)
|
||||
for d in spec.get("dirs", []):
|
||||
parts = norm(d)[2:].split("/")
|
||||
for k in range(1, len(parts) + 1): # parents first, keep existing ones
|
||||
p = norm("/".join(parts[:k]))
|
||||
if p not in t.m:
|
||||
t.put(info(p, tarfile.DIRTYPE, 0o755, mt), None)
|
||||
raw = io.BytesIO()
|
||||
with tarfile.open(fileobj=raw, mode="w", format=tarfile.GNU_FORMAT) as tf:
|
||||
for n in t.order:
|
||||
ti, data = t.m[n]
|
||||
tf.addfile(ti, io.BytesIO(data) if data is not None else None)
|
||||
with open(out, "wb") as fo, gzip.GzipFile(filename="", mode="wb", fileobj=fo, mtime=0) as gz:
|
||||
gz.write(raw.getvalue())
|
||||
n_dev = sum(1 for n in t.order if t.m[n][0].isdev())
|
||||
print(f"{len(t.order)} members ({n_dev} device nodes) -> {out}")
|
||||
print("sha256", hashlib.sha256(open(out, "rb").read()).hexdigest())
|
||||
if "--verify" in argv:
|
||||
bad = 0
|
||||
for p, want in spec.get("expect", {}).items():
|
||||
e = t.m.get(norm(p))
|
||||
got = hashlib.sha256(e[1]).hexdigest() if e and e[1] is not None else None
|
||||
ok = got is not None and got.startswith(want.lower())
|
||||
bad += not ok
|
||||
print(("OK " if ok else "BAD ") + (got or "missing")[:16], p)
|
||||
for l in spec.get("links", []):
|
||||
e = t.m.get(norm(l["path"]))
|
||||
ok = e is not None and e[0].issym() and e[0].linkname == l["target"]
|
||||
bad += not ok
|
||||
print(("OK " if ok else "BAD ") + "link", l["path"], "->", l["target"])
|
||||
if bad:
|
||||
sys.exit(f"{bad} check(s) failed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""qspi_image.py -- build a whole-flash QSPI image for a Zynq-7000 board.
|
||||
|
||||
python qspi_image.py <out.bin> <offset>:<file> [<offset>:<file> ...]
|
||||
[--erase <offset>:<len> ...] [--size <bytes>] [--layout <out.json>]
|
||||
|
||||
Every part is placed at its offset; gaps (and --erase ranges, e.g. the U-Boot
|
||||
environment sector that must stay blank so the default bootcmd is used) are
|
||||
0xFF, exactly what a fresh `sf erase` leaves. The image ends at the last byte
|
||||
of the last part (or --size). Overlapping parts are refused. Prints
|
||||
offset / length / CRC32 of every part and of the image, the numbers to check
|
||||
against `crc32` in U-Boot, and writes them to --layout as JSON.
|
||||
Offsets and lengths accept 0x hex.
|
||||
"""
|
||||
import json, sys, zlib
|
||||
|
||||
def num(s): return int(s, 0)
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
out, rest = argv[1], argv[2:]
|
||||
parts, erase, size, layout = [], [], None, None
|
||||
i = 0
|
||||
while i < len(rest):
|
||||
a = rest[i]
|
||||
if a == "--erase":
|
||||
o, l = rest[i + 1].split(":"); erase.append((num(o), num(l))); i += 2
|
||||
elif a == "--size":
|
||||
size = num(rest[i + 1]); i += 2
|
||||
elif a == "--layout":
|
||||
layout = rest[i + 1]; i += 2
|
||||
else:
|
||||
o, f = a.split(":", 1); parts.append((num(o), f)); i += 1
|
||||
blobs = []
|
||||
for o, f in sorted(parts):
|
||||
d = open(f, "rb").read()
|
||||
blobs.append((o, f, d))
|
||||
for (o1, f1, d1), (o2, f2, d2) in zip(blobs, blobs[1:]):
|
||||
if o1 + len(d1) > o2:
|
||||
sys.exit(f"overlap: {f1} 0x{o1:x}+0x{len(d1):x} runs into {f2} at 0x{o2:x}")
|
||||
end = max(o + len(d) for o, _, d in blobs) # erased tails need no bytes: blank anyway
|
||||
if size is not None:
|
||||
if size < end:
|
||||
sys.exit(f"--size 0x{size:x} smaller than the content end 0x{end:x}")
|
||||
end = size
|
||||
img = bytearray(b"\xff" * end)
|
||||
for o, _, d in blobs:
|
||||
img[o:o + len(d)] = d
|
||||
for o, l in erase:
|
||||
for o2, f2, d2 in blobs:
|
||||
if o < o2 + len(d2) and o2 < o + l:
|
||||
sys.exit(f"--erase 0x{o:x}:0x{l:x} overlaps {f2}")
|
||||
open(out, "wb").write(img)
|
||||
rep = {"image": out, "length": len(img), "crc32": "%08x" % zlib.crc32(img), "parts": [],
|
||||
"erased": [{"offset": hex(o), "length": hex(l)} for o, l in erase]}
|
||||
for o, f, d in blobs:
|
||||
rep["parts"].append({"file": f, "offset": hex(o), "length": hex(len(d)),
|
||||
"crc32": "%08x" % zlib.crc32(d)})
|
||||
print(f"0x{o:08x} len 0x{len(d):x} crc32 {zlib.crc32(d):08x} {f}")
|
||||
for o, l in erase:
|
||||
print(f"0x{o:08x} len 0x{l:x} erased (0xFF)")
|
||||
print(f"image {out}: len 0x{len(img):x} crc32 {rep['crc32']}")
|
||||
if layout:
|
||||
json.dump(rep, open(layout, "w"), indent=1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv)
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""release_check.py -- checksum, verify and scan a release directory.
|
||||
|
||||
python release_check.py sums <release_dir> # (re)write 00_SHA256SUMS
|
||||
python release_check.py verify <release_dir> # check every listed file
|
||||
python release_check.py scan <release_dir> [--env ~/skills_env.md] [--mac AA:BB:..]...
|
||||
[--src-dirs 07_ps_app_src,08_imx6_app_src] [--allow KEY]...
|
||||
|
||||
scan reports (never prints a secret value, only the file and the key name):
|
||||
* values of *PASS* / *PASSWORD* / *TOKEN* / *SECRET* / *KEY* keys in --env found
|
||||
in any text file, and "user:password@" URLs; --allow KEY moves that key's hits
|
||||
to "acknowledged" (e.g. a vendor factory-default account that ships in the
|
||||
firmware and is also the brand name) -- still listed, no failure;
|
||||
* tool state files (*.upstate.json) inside --src-dirs;
|
||||
* private keys (-----BEGIN ... PRIVATE KEY-----);
|
||||
* the given board MACs (--mac, e.g. a test board's) and any other MAC literal;
|
||||
* development-machine absolute paths (X:\\..., /home/<user>, /Users/<user>);
|
||||
* ELF / object / archive files inside --src-dirs (build output in a source tree);
|
||||
* Windows reserved names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) that break copies.
|
||||
Exit status 0 = scan finished (read the report), 2 = a secret was found.
|
||||
"""
|
||||
import hashlib, os, re, sys
|
||||
|
||||
SUMS = "00_SHA256SUMS"
|
||||
RESERVED = re.compile(r"^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$", re.I)
|
||||
MAC = re.compile(r"\b[0-9A-Fa-f]{2}(?::[0-9A-Fa-f]{2}){5}\b")
|
||||
WINPATH = re.compile(r"\b[A-Za-z]:[\\/](?:[^\s\"'<>|]+)")
|
||||
NIXHOME = re.compile(r"/(?:home|Users)/[A-Za-z0-9_.-]+")
|
||||
URLCRED = re.compile(r"[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@", re.I)
|
||||
PRIVKEY = re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")
|
||||
|
||||
|
||||
def files(root):
|
||||
for d, _, fs in os.walk(root):
|
||||
for f in fs:
|
||||
yield os.path.join(d, f)
|
||||
|
||||
|
||||
def relp(root, p):
|
||||
return "./" + os.path.relpath(p, root).replace(os.sep, "/")
|
||||
|
||||
|
||||
def sha(p):
|
||||
h = hashlib.sha256()
|
||||
with open(p, "rb") as f:
|
||||
for b in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def is_text(b):
|
||||
return b"\0" not in b[:8192]
|
||||
|
||||
|
||||
def cmd_sums(root):
|
||||
lines = [f"{sha(p)} {relp(root, p)}" for p in sorted(files(root))
|
||||
if os.path.basename(p) != SUMS]
|
||||
open(os.path.join(root, SUMS), "w", newline="\n").write("\n".join(lines) + "\n")
|
||||
print(f"{len(lines)} files -> {SUMS}")
|
||||
|
||||
|
||||
def cmd_verify(root):
|
||||
bad = n = 0
|
||||
listed = set()
|
||||
for line in open(os.path.join(root, SUMS)):
|
||||
h, p = line.rstrip("\n").split(" ", 1)
|
||||
listed.add(p); n += 1
|
||||
fp = os.path.join(root, p[2:])
|
||||
if not os.path.exists(fp) or sha(fp) != h:
|
||||
bad += 1; print("BAD", p)
|
||||
extra = [relp(root, p) for p in files(root)
|
||||
if os.path.basename(p) != SUMS and relp(root, p) not in listed]
|
||||
for p in extra:
|
||||
print("NOT LISTED", p)
|
||||
print(f"{n} listed, {bad} bad, {len(extra)} not listed")
|
||||
sys.exit(1 if bad or extra else 0)
|
||||
|
||||
|
||||
def env_secrets(path):
|
||||
out = {}
|
||||
if not path or not os.path.exists(path):
|
||||
return out
|
||||
for line in open(path, encoding="utf-8", errors="replace"):
|
||||
m = re.match(r"\s*([A-Z0-9_]*(?:PASS|PASSWORD|TOKEN|SECRET|KEY)[A-Z0-9_]*)\s*=\s*(\S+)", line)
|
||||
if m and len(m.group(2)) >= 4:
|
||||
out[m.group(1)] = m.group(2)
|
||||
return out
|
||||
|
||||
|
||||
def cmd_scan(root, args):
|
||||
env = os.path.expanduser(args[args.index("--env") + 1]) if "--env" in args else None
|
||||
macs = [args[i + 1].lower() for i, a in enumerate(args) if a == "--mac"]
|
||||
srcd = args[args.index("--src-dirs") + 1].split(",") if "--src-dirs" in args else []
|
||||
allow = {args[i + 1] for i, a in enumerate(args) if a == "--allow"}
|
||||
secrets = env_secrets(env)
|
||||
found_secret = False
|
||||
rep = {k: [] for k in ("secret", "acknowledged", "url_cred", "privkey", "board_mac", "mac", "dev_path",
|
||||
"src_binary", "reserved")}
|
||||
for p in files(root):
|
||||
r = relp(root, p)
|
||||
if RESERVED.match(os.path.basename(p)):
|
||||
rep["reserved"].append(r)
|
||||
b = open(p, "rb").read()
|
||||
if any(r.startswith("./" + s.strip("/") + "/") for s in srcd):
|
||||
if p.endswith(".upstate.json"):
|
||||
rep["src_binary"].append(r + " (tool state file, remove)")
|
||||
elif b[:4] == b"\x7fELF" or p.endswith((".o", ".a", ".so", ".ko")):
|
||||
rep["src_binary"].append(r)
|
||||
if not is_text(b):
|
||||
continue
|
||||
t = b.decode("utf-8", "replace")
|
||||
for k, v in secrets.items():
|
||||
if v in t:
|
||||
if k in allow:
|
||||
rep["acknowledged"].append(f"{r} (value of {k})")
|
||||
else:
|
||||
rep["secret"].append(f"{r} (value of {k})"); found_secret = True
|
||||
if URLCRED.search(t):
|
||||
rep["url_cred"].append(r); found_secret = True
|
||||
if PRIVKEY.search(t):
|
||||
rep["privkey"].append(r)
|
||||
ms = {m.lower() for m in MAC.findall(t)}
|
||||
for m in ms:
|
||||
(rep["board_mac"] if m in macs else rep["mac"]).append(f"{r} {m}")
|
||||
dp = sorted(set(WINPATH.findall(t)) | set(NIXHOME.findall(t)))
|
||||
if dp:
|
||||
rep["dev_path"].append(f"{r} ({len(dp)}: {dp[0][:60]}{' ...' if len(dp) > 1 else ''})")
|
||||
titles = {"secret": "SECRET VALUES (must be removed)",
|
||||
"acknowledged": "acknowledged values (--allow; say so in the README)",
|
||||
"url_cred": "user:password@ URLs",
|
||||
"privkey": "private keys (check: product key or leaked?)",
|
||||
"board_mac": "given board MACs (must not ship)", "mac": "other MAC literals (review)",
|
||||
"dev_path": "development-machine paths (review / document)",
|
||||
"src_binary": "binaries inside source dirs", "reserved": "Windows reserved names"}
|
||||
for k, title in titles.items():
|
||||
print(f"== {title}: {len(rep[k])}")
|
||||
for x in rep[k][:40]:
|
||||
print(" ", x)
|
||||
if len(rep[k]) > 40:
|
||||
print(f" ... {len(rep[k]) - 40} more")
|
||||
sys.exit(2 if found_secret else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
sys.exit(__doc__)
|
||||
c, root = sys.argv[1], sys.argv[2]
|
||||
{"sums": lambda: cmd_sums(root), "verify": lambda: cmd_verify(root),
|
||||
"scan": lambda: cmd_scan(root, sys.argv[3:])}[c]()
|
||||
Reference in New Issue
Block a user