#!/usr/bin/env -S python3 -I """The craft-libs build loop: cargo with one resolution of features, so every local build shares one build of each crate. cl check [cargo args] cargo check of the crate's lib, bins and tests cl test [cargo args] [-- test args] build the crate's tests (cargo test --no-run), then run each test binary cl build gallery [cargo args] the debug gallery binary (target/debug/craft-ui-gallery) cl build [cargo args] cargo build of the crate's lib and bins cl render [--card] [--story ] [--out ] [--threads N] [--theme ] [--mode light|dark] [--density compact|comfortable] [--contrast standard|high] the component's stories (--card: its Overview cards) as PNGs through the story-render bin cl features print the --features value cl adds for cl doctor check the manifests for drift from the one feature resolution cl cargo cargo through the job pool, nothing added Every cargo command runs through the command in CL_LAUNCHER when it is set (for example a wrapper that gives several builds on one machine a shared jobserver; docs/development.md "The job pool"), else plain cargo. The binaries cl runs after a build (test binaries, the story renderer) run through the command in CL_RUNNER when it is set (for example a wrapper that limits how many run at once), else directly; a test binary runs from its package's directory, as cargo runs it. CARGO_TARGET_DIR defaults to -target when that directory exists beside the worktree. Why one resolution: cargo builds a crate once per distinct feature set, and everything above it again, and it unifies features only over the packages a command selects. A framework crate's tests, a domain crate's tests, the gallery and the story renderer each pull in different crates, which turn on different features of egui, wgpu, craft-window and others below the framework, so each would build its own copy of the framework (minutes, and wgpu again). cl selects the `feature-root` package beside every command's package: its dependencies are the gallery's, with every domain crate, so every command resolves features as the gallery does and builds only the targets it names. The crate's own test feature (`screen-testing`, else `testing`) is added on top. Builds that take other features (CI's `--all-features`, wasm checks) belong on GitHub. docs/development.md has the whole picture. Records: every build cl composes passes cargo's --timings (a report per build in /cargo-timings/, the newest 200 kept), and every cl command appends one line to /cl.log: when it started, the command, the build's and the test runs' wall time, the exit code and the build's report. Guards: `--all-features`, `--bins` and workspace-wide builds are refused with the reason; after every command, a new build of a framework crate in the target directory is named, with its feature set; `cl doctor` (also run before every command, it reads a few manifests) reports manifest lines that would bring a second build back. """ import json import os import re import shlex import subprocess import sys import time FRAMEWORK = ["craft-ui-contract", "craft-ui-widgets", "craft-ui-panels", "craft-ui-canvas", "craft-ui-patterns", "craft-ui-shell", "craft-ui"] # Selected beside every command's package; its dependencies decide every shared crate's features (its manifest). ROOT = "feature-root" # Domain crates the story renderer can link, by the crate's short name. RENDER_FEATURES = { "automation", "browse", "color", "draw", "imaging", "layers", "menu", "pages", "print", "review", "table", "text", "theme-store", "timeline", "dialogs", "font-manager", "update", } REFUSED = { "--all-features": "it enables snapshot, system_fonts, color_fonts, all-languages and more, a set no loop uses, which rebuilds every dependency and the whole framework once; CI runs it", "--bins": "the gallery's binaries are 1 GB links each; build the one you need (`cl build gallery`)", "--workspace": "a workspace-wide build or test takes the better part of an hour; build the crate you changed, CI runs the rest", "--all": "a workspace-wide build or test takes the better part of an hour; build the crate you changed, CI runs the rest", "--all-targets": "it builds every binary and benchmark too; use `cl check` or `cl test` (--tests)", } TIMING_REPORTS_KEPT = 200 # What cl.log records about this command; filled in as it runs. RECORD = {"start": time.time(), "target": None, "build": None, "run": 0.0, "report": ""} # The manifest lines the one feature set rests on: (file, regex that must match, what its absence brings back). PINS = [ ("crates/ui-macros/Cargo.toml", r'^syn = \{[^\n]*"full"[^\n]*"visit"[^\n]*"visit-mut"[^\n]*"fold"[^\n]*"extra-traits"', "craft-ui-macros must ask syn for full, visit, visit-mut, fold and extra-traits, or any derive crate in a build changes the shared syn and rebuilds the framework"), ("crates/ui-macros/Cargo.toml", r'^proc-macro2 = \{[^\n]*"span-locations"', "craft-ui-macros must ask proc-macro2 for span-locations, or a build with design-lint in it changes the shared proc-macro2 and rebuilds the framework"), ("crates/ui-contract/Cargo.toml", r'^craft-window = \{[^\n]*features = \[[^\]]*"ui"', "craft-ui-contract must depend on craft-window with ui, or the shell's app flips craft-window under the whole framework"), ("crates/ui-contract/Cargo.toml", r'^testing = \[[^\]]*"snapshot"', "craft-ui-contract's testing must include snapshot, or crates above widgets cannot ask the runtime for the renderer and build a second variant"), ] def fail(message): print(f"cl: {message}", file=sys.stderr) sys.exit(2) def worktree(): top = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True) if top.returncode != 0: fail("not inside a git worktree") return top.stdout.strip() def metadata(top): out = subprocess.run( ["cargo", "metadata", "--no-deps", "--format-version", "1", "--offline"], cwd=top, capture_output=True, text=True ) if out.returncode != 0: fail("cargo metadata failed:\n" + out.stderr) return {p["name"]: p for p in json.loads(out.stdout)["packages"]} def package(packages, name): if name in packages: return packages[name] if "craft-" + name in packages: return packages["craft-" + name] fail(f"no workspace package named {name}") def canonical(packages, name): """The package's name and the features cl adds for it, qualified: its own test feature.""" p = package(packages, name) if "screen-testing" in p["features"]: return p["name"], [p["name"] + "/screen-testing"] if "testing" in p["features"] and p["name"] != "craft-ui-test-kit": return p["name"], [p["name"] + "/testing"] return p["name"], [] def own_targets(p): """Target-selection flags naming the package's lib and bins, so the root's bin is never selected.""" flags = [] for t in p["targets"]: if "lib" in t["kind"] or "rlib" in t["kind"] or "proc-macro" in t["kind"]: flags.append("--lib") elif "bin" in t["kind"]: flags += ["--bin", t["name"]] return flags def doctor(top, packages, quiet=False): """Manifest lines that would bring the variant churn back. Returns the number of problems.""" problems = [] for path, pattern, why in PINS: text = open(os.path.join(top, path)).read() if not re.search(pattern, text, re.M): problems.append(f"{path}: {why}") for p in packages.values(): manifest = os.path.relpath(p["manifest_path"], top) for d in p["dependencies"]: if d["name"] == "egui" and d["kind"] == "dev" and d["features"]: problems.append(f"{manifest}: the egui dev-dependency asks for {d['features']}; a feature on a dev-dependency " "line changes egui under the framework between a crate's tests and its render") gallery = packages.get("craft-ui-gallery") if gallery: for d in gallery["dependencies"]: if d["name"].startswith("craft-") and "screen-testing" in d["features"] and not d["optional"]: problems.append(f"crates/ui-gallery/Cargo.toml: {d['name']} must be optional (one feature per domain crate), or the story renderer links every crate") root = packages.get(ROOT) if not root: problems.append(f"crates/{ROOT}: missing; without it every kind of command builds its own copy of the framework") elif gallery: mine = {d["name"]: d for d in root["dependencies"] if d["kind"] is None} g = mine.get("craft-ui-gallery") if not g or g["uses_default_features"] or g["features"]: problems.append(f"crates/{ROOT}/Cargo.toml: must depend on craft-ui-gallery with default-features = false and no " "features, or every render links all domain crates") for d in gallery["dependencies"]: if d["kind"] is None and d["optional"] and d["name"].startswith("craft-"): r = mine.get(d["name"]) if not r or not set(d["features"]) <= set(r["features"]): problems.append(f"crates/{ROOT}/Cargo.toml: must depend on {d['name']} with {d['features']} as the gallery " "does, or a domain crate's tests and the gallery build different copies of the framework") render = packages.get("story-render") if gallery and render: theirs = {d["name"]: d for d in gallery["dependencies"] if d["kind"] is None} for d in render["dependencies"]: g = theirs.get(d["name"]) if d["kind"] is None and g and not set(d["features"]) <= set(g["features"]): problems.append(f"crates/story-render/Cargo.toml: {d['name']} asks for {sorted(set(d['features']) - set(g['features']))} " "beyond the gallery's line, so renders build a second copy of it and everything above") for problem in problems: print(f"cl doctor: {problem}", file=sys.stderr) if not problems and not quiet: print("cl doctor: the manifests keep the one feature set", file=sys.stderr) return len(problems) def target_dir(top, env): if "CARGO_TARGET_DIR" not in env and os.path.isdir(top + "-target"): env["CARGO_TARGET_DIR"] = top + "-target" return env.get("CARGO_TARGET_DIR", os.path.join(top, "target")) def variants(target): """The framework crates' lib builds in the target dir's debug profile: {(crate, hash): (features, mode)}, mode "check" for a metadata-only build (`cargo check`, no rlib) and "build" otherwise.""" found = {} fp = os.path.join(target, "debug", ".fingerprint") if not os.path.isdir(fp): return found for d in os.listdir(fp): m = re.match(r"(craft-ui(?:-[a-z]+)?)-([0-9a-f]{16})$", d) if not m or m.group(1) not in FRAMEWORK: continue for f in os.listdir(os.path.join(fp, d)): if f.startswith("lib-") and f.endswith(".json"): try: j = json.load(open(os.path.join(fp, d, f))) except (OSError, ValueError): continue rlib = os.path.join(target, "debug", "deps", f"lib{m.group(1).replace('-', '_')}-{m.group(2)}.rlib") found[(m.group(1), m.group(2))] = (j.get("features", ""), "build" if os.path.exists(rlib) else "check") return found def report_new_variants(before, after): """Names each new build of a framework crate. A check and a build of a crate are separate by design; within each, a target dir should hold one.""" new = {k: v for k, v in after.items() if k not in before} for crate in FRAMEWORK: for mode in ("build", "check"): mine = sorted({f for (c, _), (f, m) in new.items() if c == crate and m == mode}) if not mine: continue olds = {f for (c, _), (f, m) in before.items() if c == crate and m == mode} if not olds: continue why = ("same features as one already here, so a crate below it changed its features (a dev-dependency line, " "a manifest change, or a command run outside cl)" if any(f in olds for f in mine) else "a feature set this target dir had not built") print(f"cl: another {mode} of {crate}, features {' | '.join(mine)}: {why}; {len(olds)} were there before. " "Expected once per target dir after a manifest change (docs/development.md); otherwise run `cl doctor`.", file=sys.stderr) def timing_report(target, since): """The timing report cargo wrote after `since`, and the older ones beyond the newest TIMING_REPORTS_KEPT removed.""" folder = os.path.join(target, "cargo-timings") try: reports = sorted((e.stat().st_mtime, e.path) for e in os.scandir(folder) if e.name.startswith("cargo-timing-")) except OSError: return "" for _, path in reports[:-TIMING_REPORTS_KEPT]: try: os.remove(path) except OSError: pass return os.path.basename(reports[-1][1]) if reports and reports[-1][0] >= since else "" def log_command(rc): """One line in /cl.log for this cl command.""" if not RECORD["target"]: return took = lambda seconds: "-" if seconds is None else f"{seconds:.1f} s" line = (f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(RECORD['start']))} cl {' '.join(sys.argv[1:])} | " f"build {took(RECORD['build'])} | run {took(RECORD['run'])} | rc {rc} | {RECORD['report'] or '-'}\n") try: with open(os.path.join(RECORD["target"], "cl.log"), "a") as f: f.write(line) except OSError: pass def cargo(top, cargo_args, json_out=False, timings=True): """Runs cargo through CL_LAUNCHER, with a timing report unless `timings` is off, and reports new framework builds. Returns (return code, target dir, the JSON messages cargo printed on stdout when `json_out`).""" # cargo fmt builds nothing, so its `--all` is the usual whole-workspace format. for arg in ([] if cargo_args[:1] == ["fmt"] else cargo_args): if arg in REFUSED: fail(f"{arg} refused: {REFUSED[arg]}") env = dict(os.environ) target = target_dir(top, env) RECORD["target"] = target if timings: cargo_args = cargo_args[:1] + ["--timings"] + cargo_args[1:] cmd = shlex.split(env.get("CL_LAUNCHER", "")) + ["cargo"] + cargo_args print("cl: " + " ".join(cmd) + f" (target {target})", file=sys.stderr) sys.stdout.flush() before = variants(target) started = time.time() messages = [] if json_out: proc = subprocess.Popen(cmd, cwd=top, env=env, stdout=subprocess.PIPE, text=True) for line in proc.stdout: try: messages.append(json.loads(line)) except ValueError: sys.stdout.write(line) rc = proc.wait() else: rc = subprocess.run(cmd, cwd=top, env=env).returncode RECORD["build"] = time.time() - started if timings: RECORD["report"] = timing_report(target, started) report_new_variants(before, variants(target)) return rc, target, messages def run_binary(cmd, cwd, env=None): """Runs a built binary through CL_RUNNER.""" sys.stdout.flush() started = time.time() rc = subprocess.run(shlex.split(os.environ.get("CL_RUNNER", "")) + cmd, cwd=cwd, env=env).returncode RECORD["run"] += time.time() - started return rc def shown(path, top): """A path as cargo shows it: relative to the worktree when inside it.""" return os.path.relpath(path, top) if path.startswith(top + os.sep) else path def test(top, packages, cargo_args, test_args): """Builds the tests with `cargo test --no-run`, then runs each test binary as cargo would: from its package's directory, the lib's unit tests first, stopping at the first that fails.""" no_run = "--no-run" in cargo_args quiet = "-q" in cargo_args or "--quiet" in cargo_args cargo_args = [a for a in cargo_args if a != "--no-run"] rc, target, messages = cargo(top, cargo_args[:1] + ["--no-run", "--message-format=json-render-diagnostics"] + cargo_args[1:], json_out=True) if rc != 0: sys.exit(rc) by_manifest = {p["manifest_path"]: p for p in packages.values()} tests = [] for m in messages: if m.get("reason") != "compiler-artifact" or not m.get("executable") or not m["profile"]["test"]: continue root = os.path.dirname(m["manifest_path"]) source = os.path.relpath(m["target"]["src_path"], root) lib = any(k in ("lib", "rlib", "proc-macro") for k in m["target"]["kind"]) tests.append((not lib, m["target"]["name"], "unittests " + source if lib else source, m["executable"], m["manifest_path"])) tests.sort() for _, _, source, exe, manifest in tests: print(f"{'Executable':>12} {source} ({shown(exe, top)})", file=sys.stderr) if no_run: sys.exit(0) for _, _, source, exe, manifest in tests: if not quiet: print(f"{'Running':>12} {source} ({shown(exe, top)})", file=sys.stderr) package = by_manifest.get(manifest, {}) env = dict(os.environ, CARGO_MANIFEST_DIR=os.path.dirname(manifest), CARGO_PKG_NAME=package.get("name", "")) rc = run_binary([exe] + test_args + (["--quiet"] if quiet else []), os.path.dirname(manifest), env) if rc != 0: print(f"cl: test failed: {source} ({shown(exe, top)}) exited with {rc}", file=sys.stderr) sys.exit(rc) sys.exit(0) def run(top, cargo_args, then=None, timings=True): """Runs cargo; `then` (a binary in the target's debug dir and its arguments) runs after a successful cargo. Exits with the last return code.""" rc, target, _ = cargo(top, cargo_args, timings=timings) if rc == 0 and then: rc = run_binary([os.path.join(target, "debug", then[0])] + then[1:], top) sys.exit(rc) def with_features(args, features): return args + (["--features", ",".join(features)] if features else []) def selected(pkg): return ["-p", pkg, "-p", ROOT] def main(): if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): print(__doc__.strip()) sys.exit(0) command, rest = sys.argv[1], sys.argv[2:] top = worktree() packages = metadata(top) if command == "doctor": sys.exit(1 if doctor(top, packages) else 0) doctor(top, packages, quiet=True) if command == "cargo": run(top, rest, timings=False) if command == "features": if not rest: fail("cl features ") print(",".join(canonical(packages, rest[0])[1])) return if command in ("check", "test", "build"): if not rest: fail(f"cl {command} [args]") name, args = rest[0], rest[1:] if command == "build" and name == "gallery": run(top, ["build"] + selected("craft-ui-gallery") + ["--bin", "craft-ui-gallery"] + args) pkg, features = canonical(packages, name) if command == "check": run(top, with_features(["check"] + selected(pkg) + ["--tests"] + own_targets(packages[pkg]), features) + args) if command == "test": mine, theirs = (args[:args.index("--")], args[args.index("--") + 1:]) if "--" in args else (args, []) test(top, packages, with_features(["test"] + selected(pkg) + ["--tests"], features) + mine, theirs) run(top, with_features(["build"] + selected(pkg) + own_targets(packages[pkg]), features) + args) if command == "render": if len(rest) < 2: fail("cl render [--card] [--story ] [--out ] [--threads N] [--theme ] [--mode light|dark] [--density compact|comfortable] [--contrast standard|high]") crate, component, options = rest[0], rest[1], rest[2:] short = crate.removeprefix("craft-") features = [] if short != "ui": if short not in RENDER_FEATURES: fail(f"{crate} is not a crate the story renderer links; one of: ui, " + ", ".join(sorted(RENDER_FEATURES))) features = ["story-render/" + short] if "--out" not in options: options += ["--out", os.path.join(os.environ.get("CL_RENDER_DIR", os.path.expanduser("~/.cache/story-render")), component)] # `cargo run` takes one package, so the bin is built with the root selected and then run. run(top, with_features(["build", "-q"] + selected("story-render") + ["--bin", "story-render"], features), then=["story-render", "--only", component] + options) fail(f"unknown command {command}; see cl --help") if __name__ == "__main__": try: main() except SystemExit as done: log_command(done.code if isinstance(done.code, int) else 1) raise