feat(driver): embed native compilation and link resources

Link LLVM, the Clang driver and LLD statically, and embed native link inputs
so executable generation needs no installed development toolchain.
Keep glibc dynamically linked on Linux.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

3 files changed, 374 insertions, 0 deletions

diff --git a/tools/driver/NativeCompiler.cc b/tools/driver/NativeCompiler.cc
new file mode 100644
index 0000000..f2ea3dc
--- /dev/null
+++ b/tools/driver/NativeCompiler.cc
@@ -0,0 +1,197 @@
+#include "NativeCompiler.h"
+#include "cps/CodeGen.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Driver/Compilation.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Job.h"
+#include "lld/Common/Driver.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/PassRegistry.h"
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Compression.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include <cstdlib>
+
+#ifdef _WIN32
+LLD_HAS_DRIVER(mingw)
+LLD_HAS_DRIVER(coff)
+#elif defined(__APPLE__)
+LLD_HAS_DRIVER(macho)
+#else
+LLD_HAS_DRIVER(elf)
+#endif
+
+namespace {
+
+#include "NativeRuntime.inc"
+
+struct TemporaryDirectory {
+    llvm::SmallString<256> Path;
+    ~TemporaryDirectory() {
+        if (!Path.empty()) llvm::sys::fs::remove_directories(Path);
+    }
+};
+
+bool fail(const llvm::Twine &Message) {
+    llvm::errs() << "cpsc: fatal error: " << Message << "\n";
+    return false;
+}
+
+class DriverDiagnostics : public clang::DiagnosticConsumer {
+    void HandleDiagnostic(clang::DiagnosticsEngine::Level Level,
+                          const clang::Diagnostic &Info) override {
+        clang::DiagnosticConsumer::HandleDiagnostic(Level, Info);
+        llvm::SmallString<256> Message;
+        Info.FormatDiagnostic(Message);
+        llvm::errs() << "cpsc: " << (Level >= clang::DiagnosticsEngine::Error
+                                        ? "error: " : "warning: ") << Message << "\n";
+    }
+};
+
+} // namespace
+
+bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath) {
+    TemporaryDirectory Runtime;
+    if (auto EC = llvm::sys::fs::createUniqueDirectory("cpsc-native", Runtime.Path))
+        return fail("cannot create native compilation directory: " + EC.message());
+
+    auto Manifest = llvm::json::parse(NativeRuntimeManifest);
+    if (!Manifest) return fail(llvm::toString(Manifest.takeError()));
+    auto &Metadata = *Manifest->getAsObject();
+    llvm::SmallVector<uint8_t, 0> Data;
+    if (auto Error = llvm::compression::zlib::decompress(
+            llvm::ArrayRef(reinterpret_cast<const uint8_t *>(NativeRuntimeData),
+                           sizeof(NativeRuntimeData) - 1), Data, NativeRuntimeSize))
+        return fail("cannot unpack native link resources: " + llvm::toString(std::move(Error)));
+
+    auto relocate = [&](llvm::StringRef Text) {
+        std::string Result = Text.str();
+        const std::string Marker = "@CPSC_RUNTIME@";
+        size_t Position = 0;
+        while ((Position = Result.find(Marker, Position)) != std::string::npos) {
+            Result.replace(Position, Marker.size(), Runtime.Path.str());
+            Position += Runtime.Path.size();
+        }
+        return Result;
+    };
+    for (const auto &Entry : *Metadata.getArray("files")) {
+        const auto &File = *Entry.getAsObject();
+        llvm::SmallString<256> Path(Runtime.Path);
+        llvm::sys::path::append(Path, *File.getString("path"));
+        if (auto EC = llvm::sys::fs::create_directories(llvm::sys::path::parent_path(Path)))
+            return fail("cannot prepare native link resources: " + EC.message());
+        std::error_code EC;
+        llvm::raw_fd_ostream Output(Path, EC);
+        if (EC) return fail("cannot open native link resource: " + EC.message());
+        llvm::StringRef Contents(reinterpret_cast<const char *>(Data.data()) +
+                                     *File.getInteger("offset"), *File.getInteger("size"));
+        if (*File.getBoolean("script")) Output << relocate(Contents);
+        else Output << Contents;
+        Output.close();
+        if (Output.has_error()) {
+            auto Error = Output.error();
+            Output.clear_error();
+            return fail("cannot write native link resource: " + Error.message());
+        }
+    }
+
+    if (llvm::InitializeNativeTarget() || llvm::InitializeNativeTargetAsmPrinter())
+        return fail("cannot initialize the native LLVM target");
+    auto &Registry = *llvm::PassRegistry::getPassRegistry();
+    llvm::initializeCore(Registry);
+    llvm::initializeCodeGen(Registry);
+    llvm::initializeScalarOpts(Registry);
+    llvm::initializeVectorization(Registry);
+    llvm::initializeTransformUtils(Registry);
+    llvm::initializeInstCombine(Registry);
+    llvm::initializeTarget(Registry);
+    llvm::Triple Triple(*Metadata.getString("target"));
+    std::string Error;
+    const auto *Target = llvm::TargetRegistry::lookupTarget(Triple, Error);
+    if (!Target) return fail(Error);
+    std::unique_ptr<llvm::TargetMachine> Machine(Target->createTargetMachine(
+        Triple, "generic", "", llvm::TargetOptions(), llvm::Reloc::PIC_));
+    if (!Machine) return fail("cannot create the native LLVM target machine");
+    auto Module = llvm::CloneModule(CG.getModule());
+    Module->setTargetTriple(Triple);
+    Module->setDataLayout(Machine->createDataLayout());
+    llvm::SmallString<256> ObjectPath(Runtime.Path);
+    llvm::sys::path::append(ObjectPath, "program.o");
+    std::error_code EC;
+    llvm::raw_fd_ostream ObjectOutput(ObjectPath, EC);
+    if (EC) return fail("cannot open native object file: " + EC.message());
+    llvm::legacy::PassManager Passes;
+    if (Machine->addPassesToEmitFile(Passes, ObjectOutput, nullptr,
+                                    llvm::CodeGenFileType::ObjectFile))
+        return fail("native target cannot emit object files");
+    Passes.run(*Module);
+    ObjectOutput.close();
+    if (ObjectOutput.has_error()) {
+        auto Error = ObjectOutput.error();
+        ObjectOutput.clear_error();
+        return fail("cannot write native object file: " + Error.message());
+    }
+
+    // Link beside the destination so the final rename stays on one filesystem.
+    llvm::SmallString<256> StagedPath;
+    int FD;
+    if (auto EC = llvm::sys::fs::createUniqueFile(OutputPath + ".tmp-%%%%%%", FD, StagedPath))
+        return fail("cannot open native output file: " + EC.message());
+    llvm::FileRemover StagedOutput(StagedPath);
+    llvm::raw_fd_ostream CloseFD(FD, true);
+    CloseFD.close();
+
+    // All link inputs are explicit. Clang's driver provides job validation and
+    // platform command construction; no compiler subprocess is executed.
+    clang::DiagnosticOptions Options;
+    DriverDiagnostics Printer;
+    clang::DiagnosticsEngine Diagnostics(new clang::DiagnosticIDs, Options, &Printer, false);
+    clang::driver::Driver Driver("cpsc", Triple.str(), Diagnostics);
+    // The standard linker flavor constructs a job without checking for an
+    // installed linker. Its platform arguments are passed to embedded LLD.
+    std::vector<std::string> Args = {"cpsc", "--driver-mode=g++", "-nostdlib", "-nostdlib++",
+        "-fuse-ld=ld", "-o", std::string(StagedPath), std::string(ObjectPath)};
+    for (const auto &Arg : *Metadata.getArray("arguments")) {
+        Args.push_back("-Xlinker");
+        Args.push_back(relocate(*Arg.getAsString()));
+    }
+    std::vector<const char *> Argv;
+    for (const auto &Arg : Args) Argv.push_back(Arg.c_str());
+    Driver.setCheckInputsExist(false);
+    std::unique_ptr<clang::driver::Compilation> Compilation(Driver.BuildCompilation(Argv));
+    if (!Compilation || Diagnostics.hasErrorOccurred()) return false;
+    if (Compilation->getJobs().size() != 1)
+        return fail("expected one embedded linker job");
+    const auto &Job = *Compilation->getJobs().begin();
+    llvm::SmallVector<const char *, 64> LinkArgs;
+#ifdef _WIN32
+    LinkArgs.push_back("ld.lld");
+    const lld::DriverDef Linkers[] = {{lld::MinGW, &lld::mingw::link}, {lld::WinLink, &lld::coff::link}};
+#elif defined(__APPLE__)
+    LinkArgs.push_back("ld64.lld");
+    const lld::DriverDef Linkers[] = {{lld::Darwin, &lld::macho::link}};
+#else
+    LinkArgs.push_back("ld.lld");
+    const lld::DriverDef Linkers[] = {{lld::Gnu, &lld::elf::link}};
+#endif
+    LinkArgs.append(Job.getArguments().begin(), Job.getArguments().end());
+    auto Result = lld::lldMain(LinkArgs, llvm::outs(), llvm::errs(), Linkers);
+    if (!Result.canRunAgain) std::exit(1);
+    if (Result.retCode != 0) return fail("native linking failed");
+    if (auto EC = llvm::sys::fs::setPermissions(StagedPath,
+            llvm::sys::fs::all_read | llvm::sys::fs::owner_write | llvm::sys::fs::all_exe))
+        return fail("cannot set executable permissions: " + EC.message());
+    if (auto EC = llvm::sys::fs::rename(StagedPath, OutputPath))
+        return fail("cannot save native output: " + EC.message());
+    return true;
+}
diff --git a/tools/driver/NativeCompiler.h b/tools/driver/NativeCompiler.h
new file mode 100644
index 0000000..17933e9
--- /dev/null
+++ b/tools/driver/NativeCompiler.h
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "llvm/ADT/StringRef.h"
+
+namespace cps { class CodeGen; }
+
+// Generate a host executable without an external compiler, linker or SDK.
+// Failure leaves any existing output intact and reports a diagnostic.
+bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath);
diff --git a/tools/driver/prepare-native-runtime.py b/tools/driver/prepare-native-runtime.py
new file mode 100644
index 0000000..8463eba
--- /dev/null
+++ b/tools/driver/prepare-native-runtime.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Freeze the build toolchain's native link inputs into the compiler."""
+
+import argparse
+import json
+from pathlib import Path
+import re
+import shlex
+import subprocess
+import zlib
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--compiler", required=True)
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--platform", choices=("linux", "windows", "macos"), required=True)
+    parser.add_argument("--flag", action="append", default=[])
+    args = parser.parse_args()
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+
+    def compiler(*options):
+        return subprocess.run([args.compiler, *args.flag, *options], check=True,
+                              capture_output=True, text=True)
+
+    # An actual object lets the driver validate its inputs without linking a probe.
+    source = output.parent / "native-probe.cc"
+    obj = output.parent / "native-probe.o"
+    binary = output.parent / "native-probe.out"
+    source.write_text("int main() { return 0; }\n")
+    compiler("-c", str(source), "-o", str(obj))
+    static = [] if args.platform == "macos" else (
+        ["-static"] if args.platform == "windows" else
+        ["-static-libstdc++", "-static-libgcc"])
+    trace = compiler("-###", "-fuse-ld=lld", *static, str(obj), "-o", str(binary))
+    commands = [shlex.split(line.strip()) for line in trace.stderr.splitlines()
+                if line.lstrip().startswith('"')]
+    if len(commands) != 1:
+        raise RuntimeError("Expected one native linker command: " + trace.stderr)
+    command = commands[0][1:]
+    directories = [Path(arg[2:]) for arg in command if arg.startswith("-L")]
+    target = compiler("-dumpmachine").stdout.strip()
+    files = {}
+    scripts = set()
+    arguments = []
+    root = "@CPSC_RUNTIME@"
+
+    def resolve(name, static_only=False):
+        if name.startswith(":"):
+            names = [name[1:]]
+        elif args.platform == "macos":
+            names = ["lib" + name + suffix for suffix in (".tbd", ".dylib", ".a")]
+        elif static_only:
+            names = ["lib" + name + ".a"]
+        else:
+            names = ["lib" + name + suffix for suffix in (".dll.a", ".so", ".a")]
+        for directory in directories:
+            for filename in names:
+                path = directory / filename
+                if path.is_file():
+                    return path
+        for filename in names:
+            found = Path(compiler("-print-file-name=" + filename).stdout.strip())
+            if found.is_file():
+                return found
+        raise RuntimeError("Cannot find native link input " + name)
+
+    sdk = None
+    if args.platform == "macos":
+        if "-syslibroot" not in command:
+            raise RuntimeError("A macOS SDK is required to prepare native linking")
+        sdk = Path(command[command.index("-syslibroot") + 1])
+        directories += [sdk / "usr/lib"]
+        # System dylibs live in the dyld cache on users' machines. Embed their
+        # SDK link interfaces, not runtime dylibs or SDK headers.
+        for path in sorted((sdk / "usr/lib").rglob("*.tbd")):
+            files["sdk/" + path.relative_to(sdk).as_posix()] = path.read_bytes()
+
+    mapped = {}
+
+    def resource(path):
+        path = path.resolve()
+        if path in mapped:
+            return root + "/" + mapped[path]
+        if sdk and path.is_relative_to(sdk.resolve()):
+            name = "sdk/" + path.relative_to(sdk.resolve()).as_posix()
+        else:
+            name = "files/" + str(len(mapped)) + "/" + path.name
+        mapped[path] = name
+        data = path.read_bytes()
+        if b"GNU ld script" in data[:256]:
+            text = re.sub(r"/\*.*?\*/", "", data.decode(), flags=re.S)
+            # GNU libc/libm linker scripts contain absolute paths and nested
+            # GROUP/AS_NEEDED inputs. Relocate every referenced link input.
+            pattern = r"(?:/[^\s()]+|lib[\w.+-]+\.(?:so(?:\.[\d]+)*|a))"
+
+            def relocate(match):
+                token = match.group()
+                dependency = Path(token)
+                if not dependency.is_file():
+                    dependency = resolve(":" + token)
+                return resource(dependency)
+
+            data = re.sub(pattern, relocate, text).encode()
+            scripts.add(name)
+        files[name] = data
+        return root + "/" + name
+
+    static_only = args.platform == "windows"
+    i = 0
+    while i < len(command):
+        arg = command[i]
+        i += 1
+        if arg == "-o":
+            i += 1
+        elif arg == str(obj):
+            pass
+        elif arg == "-L":
+            i += 1
+        elif arg.startswith("-L"):
+            pass
+        elif arg in ("-dynamic-linker", "--dynamic-linker"):
+            # The interpreter must remain an operating-system path.
+            arguments += [arg, command[i]]
+            i += 1
+        elif arg == "-syslibroot":
+            arguments += [arg, root + "/sdk"]
+            i += 1
+        elif arg in ("-Bstatic", "-Bdynamic"):
+            static_only = arg == "-Bstatic"
+            arguments.append(arg)
+        elif arg.startswith("-l"):
+            arguments.append(resource(resolve(arg[2:], static_only)))
+        elif Path(arg).is_file():
+            arguments.append(resource(Path(arg)))
+        else:
+            arguments.append(arg)
+
+    blob = bytearray()
+    manifest = {"target": target, "arguments": arguments, "files": []}
+    for name, data in files.items():
+        manifest["files"].append({"path": name, "offset": len(blob),
+                                  "size": len(data), "script": name in scripts})
+        blob.extend(data)
+    compressed = zlib.compress(blob, 9)
+    # Small separate literals avoid MSVC's individual string literal limit.
+    literals = "\n".join('"' + "".join(f"\\x{b:02x}" for b in compressed[i:i+1024]) + '"'
+                         for i in range(0, len(compressed), 1024))
+    content = ("// Generated native link resources; do not edit.\n"
+               f"static constexpr size_t NativeRuntimeSize = {len(blob)};\n"
+               "static const char NativeRuntimeData[] =\n" + literals + ";\n"
+               'static const char NativeRuntimeManifest[] = R"CPSC(\n' +
+               json.dumps(manifest) + '\n)CPSC";\n')
+    if not output.exists() or output.read_text() != content:
+        output.write_text(content)
+    # Track the actual toolchain inputs, not only the generator script.
+    dependencies = [str(path) for path in mapped]
+    if sdk:
+        dependencies += [str(path) for path in (sdk / "usr/lib").rglob("*.tbd")]
+    (output.parent / "native-runtime-inputs.cmake").write_text(
+        "set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS\n" +
+        "\n".join('  [==[' + path + ']==]' for path in dependencies) + ")\n")
+    print(f"Embedded {len(files)} native link inputs ({len(compressed)} compressed bytes)")
+
+
+if __name__ == "__main__":
+    main()