diff options
| author | _ov4 <ov4@ov4.org> | 2026-10-16 07:54:06 +0000 |
|---|---|---|
| committer | _ov4 <ov4@ov4.org> | 2026-10-16 07:54:06 +0000 |
| commit | 9226e87baacece10ea4971a9588d46e30cc23492 (patch) | |
| tree | fe2729d2bf6c07366e71a6d39421ff05b239b713 | |
| parent | c4565386553ea86308a9487650d2435804893ed0 (diff) | |
feat(driver): embed native compilation and link resources
Link LLVM, the Clang driver and LLD statically, and embed native link inputs
so executable generation needs no installed development toolchain.
Keep glibc dynamically linked on Linux.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
| -rw-r--r-- | CMakeLists.txt | 26 | ||||
| -rw-r--r-- | cmake/EmbeddedLLVM.cmake | 2 | ||||
| -rw-r--r-- | cmake/NativeCompiler.cmake | 110 | ||||
| -rw-r--r-- | doc/build.md | 15 | ||||
| -rw-r--r-- | tools/driver/NativeCompiler.cc | 197 | ||||
| -rw-r--r-- | tools/driver/NativeCompiler.h | 9 | ||||
| -rw-r--r-- | tools/driver/prepare-native-runtime.py | 168 |
7 files changed, 507 insertions, 20 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 76b76a7..f875856 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,38 +65,40 @@ add_library(CPSCodeGen STATIC lib/CodeGen/FileHandler.cc ) -llvm_map_components_to_libnames(llvm_libs core support) +llvm_map_components_to_libnames(llvm_libs core support native passes) if(CPSC_RUN_MODE STREQUAL "EMBEDDED") - llvm_map_components_to_libnames(llvm_libs core support native mcjit) + llvm_map_components_to_libnames(llvm_libs core support native passes mcjit) foreach(llvm_lib IN LISTS llvm_libs) get_target_property(llvm_type ${llvm_lib} TYPE) if(NOT llvm_type STREQUAL "STATIC_LIBRARY") message(FATAL_ERROR "EMBEDDED requires static LLVM libraries: ${llvm_lib}") endif() endforeach() - include(cmake/EmbeddedLLVM.cmake) endif() +include(cmake/EmbeddedLLVM.cmake) add_executable(cpsc tools/driver/main.cc) target_compile_definitions(cpsc PRIVATE CPSC_RUN_${CPSC_RUN_MODE}) if(CPSC_RUN_MODE STREQUAL "EMBEDDED") target_sources(cpsc PRIVATE tools/driver/EmbeddedRunner.cc) - if(MINGW) - target_link_options(cpsc PRIVATE -static) - elseif(APPLE) - # LLVM's support interface can include unused Homebrew dylibs such as Z3. - target_link_options(cpsc PRIVATE -Wl,-dead_strip_dylibs) - elseif(UNIX AND NOT APPLE) - target_link_options(cpsc PRIVATE -static-libstdc++ -static-libgcc -Wl,--as-needed) - endif() endif() -target_link_libraries(cpsc +if(MINGW) + target_link_options(cpsc PRIVATE -static) +elseif(APPLE) + # LLVM's support interface can include unused Homebrew dylibs such as Z3. + target_link_options(cpsc PRIVATE -Wl,-dead_strip_dylibs) +elseif(UNIX AND NOT APPLE) + target_link_options(cpsc PRIVATE -static-libstdc++ -static-libgcc -Wl,--as-needed) +endif() +target_link_libraries(cpsc PRIVATE CPSCodeGen CPSParser CPSLexer ${llvm_libs} ) +include(cmake/NativeCompiler.cmake) + include(cmake/Manual.cmake) configure_file(cmake/Install.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/Install.cmake" @ONLY) diff --git a/cmake/EmbeddedLLVM.cmake b/cmake/EmbeddedLLVM.cmake index c7ad5d4..cb08edc 100644 --- a/cmake/EmbeddedLLVM.cmake +++ b/cmake/EmbeddedLLVM.cmake @@ -17,7 +17,7 @@ foreach(cpsc_dependency IN LISTS cpsc_support_libs) cpsc_find_library(CPSC_STATIC_${cpsc_library_name} ${cpsc_library_names}) if(NOT CPSC_STATIC_${cpsc_library_name}) message(FATAL_ERROR - "EMBEDDED requires a static archive for ${cpsc_dependency}; download or install its development package") + "cpsc requires a static archive for ${cpsc_dependency}; download or install its development package") endif() string(REPLACE "${cpsc_dependency}" "${CPSC_STATIC_${cpsc_library_name}}" cpsc_support_libs "${cpsc_support_libs}") diff --git a/cmake/NativeCompiler.cmake b/cmake/NativeCompiler.cmake new file mode 100644 index 0000000..8371ee2 --- /dev/null +++ b/cmake/NativeCompiler.cmake @@ -0,0 +1,110 @@ +# Native output is independent of the optional -r execution engine. +find_package(Clang REQUIRED CONFIG HINTS "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") +cpsc_find_dependency(PATH LLD_DIR NAMES LLDConfig.cmake PATH_SUFFIXES lib/cmake/lld cmake) +find_package(LLD REQUIRED CONFIG HINTS "${LLVM_INSTALL_PREFIX}/lib/cmake/lld") +find_package(Python3 3.9 REQUIRED COMPONENTS Interpreter) + +if(MINGW) + set(cpsc_native_platform windows) + set(cpsc_linker_libraries lldMinGW lldCOFF) +elseif(APPLE) + set(cpsc_native_platform macos) + set(cpsc_linker_libraries lldMachO) +else() + set(cpsc_native_platform linux) + set(cpsc_linker_libraries lldELF) +endif() + +set(cpsc_native_flags) +get_filename_component(cpsc_lld_prefix "${LLD_DIR}/../../.." ABSOLUTE) +if(APPLE) + set(cpsc_linker_names ld64.lld) +else() + set(cpsc_linker_names ld.lld-23 ld.lld) +endif() +find_program(CPSC_NATIVE_LINKER NAMES ${cpsc_linker_names} + HINTS "${cpsc_lld_prefix}/bin" ${CPSC_LOCAL_BIN_DIRS}) +if(NOT CPSC_NATIVE_LINKER) + message(FATAL_ERROR "Install the LLD 23 executable to prepare native link resources") +endif() +list(APPEND cpsc_native_flags "--flag=--ld-path=${CPSC_NATIVE_LINKER}") +target_link_options(cpsc PRIVATE -fuse-ld=lld "--ld-path=${CPSC_NATIVE_LINKER}") +if(CMAKE_SYSROOT) + list(APPEND cpsc_native_flags "--flag=--sysroot=${CMAKE_SYSROOT}") +endif() +if(APPLE) + if(NOT CMAKE_OSX_SYSROOT) + execute_process(COMMAND xcrun --show-sdk-path + OUTPUT_VARIABLE CMAKE_OSX_SYSROOT OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE cpsc_sdk_result) + if(NOT cpsc_sdk_result EQUAL 0) + message(FATAL_ERROR "Cannot locate the macOS SDK") + endif() + endif() + list(APPEND cpsc_native_flags "--flag=-isysroot" "--flag=${CMAKE_OSX_SYSROOT}") + if(CMAKE_OSX_DEPLOYMENT_TARGET) + list(APPEND cpsc_native_flags "--flag=-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() +endif() +set(cpsc_runtime_generator "${CMAKE_SOURCE_DIR}/tools/driver/prepare-native-runtime.py") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${cpsc_runtime_generator}") +execute_process(COMMAND "${Python3_EXECUTABLE}" "${cpsc_runtime_generator}" + "--compiler=${CMAKE_CXX_COMPILER}" + "--output=${CMAKE_BINARY_DIR}/native-runtime/NativeRuntime.inc" + "--platform=${cpsc_native_platform}" ${cpsc_native_flags} + RESULT_VARIABLE cpsc_runtime_result OUTPUT_VARIABLE cpsc_runtime_output + ERROR_VARIABLE cpsc_runtime_error) +if(NOT cpsc_runtime_result EQUAL 0) + message(FATAL_ERROR "Cannot prepare native link resources: ${cpsc_runtime_error}") +endif() +message(STATUS "${cpsc_runtime_output}") +include("${CMAKE_BINARY_DIR}/native-runtime/native-runtime-inputs.cmake") + +target_sources(cpsc PRIVATE tools/driver/NativeCompiler.cc) +target_include_directories(cpsc PRIVATE ${CLANG_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS} + "${CMAKE_BINARY_DIR}/native-runtime") +target_link_libraries(cpsc PRIVATE clangDriver ${cpsc_linker_libraries}) +# Distribution Clang/LLD archives can still name the shared LLVM umbrella +# library in their imported interfaces. Resolve those edges to static archives. +add_library(CPSCStaticLLVM INTERFACE) +foreach(cpsc_llvm_library IN LISTS LLVM_AVAILABLE_LIBS) + # Standalone tools are not driver/linker dependencies and may reference + # optional packages absent from binary SDK installations. + if(cpsc_llvm_library MATCHES "^LLVM(LineEditor|BOLT.*|Exegesis.*|OptDriver)$") + continue() + endif() + if(TARGET ${cpsc_llvm_library}) + get_target_property(cpsc_llvm_type ${cpsc_llvm_library} TYPE) + if(cpsc_llvm_type STREQUAL "STATIC_LIBRARY") + target_link_libraries(CPSCStaticLLVM INTERFACE ${cpsc_llvm_library}) + endif() + endif() +endforeach() +foreach(cpsc_import IN LISTS CLANG_EXPORTED_TARGETS LLD_EXPORTED_TARGETS LLVM_AVAILABLE_LIBS) + if(NOT TARGET ${cpsc_import}) + continue() + endif() + get_target_property(cpsc_interface ${cpsc_import} INTERFACE_LINK_LIBRARIES) + if(cpsc_interface) + list(FIND cpsc_interface LLVM cpsc_shared_llvm_index) + if(NOT cpsc_shared_llvm_index EQUAL -1) + list(REMOVE_ITEM cpsc_interface LLVM) + if(NOT cpsc_import IN_LIST LLVM_AVAILABLE_LIBS) + list(APPEND cpsc_interface CPSCStaticLLVM) + endif() + endif() + if(CPSC_STATIC_z) + list(TRANSFORM cpsc_interface REPLACE "^ZLIB::ZLIB$" "${CPSC_STATIC_z}") + endif() + if(CPSC_STATIC_zstd) + list(TRANSFORM cpsc_interface REPLACE "^zstd::libzstd.*$" "${CPSC_STATIC_zstd}") + endif() + set_property(TARGET ${cpsc_import} PROPERTY INTERFACE_LINK_LIBRARIES "${cpsc_interface}") + endif() +endforeach() +foreach(cpsc_native_library clangDriver ${cpsc_linker_libraries}) + get_target_property(cpsc_native_type ${cpsc_native_library} TYPE) + if(NOT cpsc_native_type STREQUAL "STATIC_LIBRARY") + message(FATAL_ERROR "Native output requires static ${cpsc_native_library}") + endif() +endforeach() diff --git a/doc/build.md b/doc/build.md index d686689..08b4f24 100644 --- a/doc/build.md +++ b/doc/build.md @@ -30,8 +30,7 @@ version. CMake rebuilds its extraction directory when the package set changes. If you use a different CMake build directory, download into its `deps/packages/` instead. Removing the build directory also removes its downloaded dependencies. -All three execution modes need LLVM development libraries to build, including -`NONE`. Users of an `EMBEDDED` executable do not need LLVM or `lli`. +All three execution modes need LLVM, Clang and LLD 23 development libraries, Python 3.9 or newer, and the native C/C++ development libraries to build, including `NONE`. Users of an `EMBEDDED` executable do not need LLVM or `lli`. CMake strictly requires LLVM development libraries and both C/C++ compilers to have major version 23. The compilers must be upstream Clang; GCC and AppleClang are rejected. Any 23.x release is accepted; 22 and 24 are rejected. LLVM/Clang @@ -46,7 +45,7 @@ Use a distribution repository that provides LLVM 23. Recommended build tools: ```bash sudo apt-get update -sudo apt-get install -y --no-install-recommends build-essential cmake clang-23 +sudo apt-get install -y --no-install-recommends build-essential cmake clang-23 lld-23 python3 ``` Recommended LLVM and support-package downloads: @@ -57,7 +56,7 @@ mkdir -p build/deps/packages cd build/deps/packages apt-get download \ llvm-23-dev llvm-23 llvm-23-runtime llvm-23-tools llvm-23-linker-tools \ - libllvm23 libclang-cpp23 libz3-dev libz3-4 \ + libllvm23 libclang-cpp23 libclang-23-dev liblld-23-dev libpolly-23-dev libz3-dev libz3-4 \ libffi-dev libffi8 libxml2-dev libxml2 \ zlib1g-dev zlib1g libzstd-dev libzstd1 libncurses-dev libtinfo6 ) @@ -77,7 +76,8 @@ pacman -Syu pacman -S --needed \ mingw-w64-x86_64-cmake \ mingw-w64-x86_64-make \ - mingw-w64-x86_64-clang + mingw-w64-x86_64-clang \ + mingw-w64-x86_64-python ``` Confirm that both installed compilers report major version 23: @@ -108,6 +108,7 @@ Recommended dependency downloads to the project build directory: ```bash mkdir -p build/deps/packages pacman -Sw --noconfirm --cachedir "$PWD/build/deps/packages" \ + mingw-w64-x86_64-clang \ mingw-w64-x86_64-llvm \ mingw-w64-x86_64-llvm-libs \ mingw-w64-x86_64-llvm-tools \ @@ -131,7 +132,7 @@ relocation tools with `xcode-select --install` if needed. The LLVM bottle below provides Clang 23. Install CMake with Homebrew: ```bash -brew install cmake +brew install cmake python lld ``` Recommended binary-bottle downloads to the build directory: @@ -139,7 +140,7 @@ Recommended binary-bottle downloads to the build directory: ```bash mkdir -p build/deps/packages HOMEBREW_CACHE="$PWD/build/deps/packages" brew fetch --force-bottle \ - llvm zlib zstd ncurses lz4 xz + llvm lld zlib zstd ncurses lz4 xz ``` [Homebrew's `llvm` formula](https://formulae.brew.sh/formula/llvm) currently diff --git a/tools/driver/NativeCompiler.cc b/tools/driver/NativeCompiler.cc new file mode 100644 index 0000000..f2ea3dc --- /dev/null +++ b/tools/driver/NativeCompiler.cc @@ -0,0 +1,197 @@ +#include "NativeCompiler.h" +#include "cps/CodeGen.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "lld/Common/Driver.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/PassRegistry.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include <cstdlib> + +#ifdef _WIN32 +LLD_HAS_DRIVER(mingw) +LLD_HAS_DRIVER(coff) +#elif defined(__APPLE__) +LLD_HAS_DRIVER(macho) +#else +LLD_HAS_DRIVER(elf) +#endif + +namespace { + +#include "NativeRuntime.inc" + +struct TemporaryDirectory { + llvm::SmallString<256> Path; + ~TemporaryDirectory() { + if (!Path.empty()) llvm::sys::fs::remove_directories(Path); + } +}; + +bool fail(const llvm::Twine &Message) { + llvm::errs() << "cpsc: fatal error: " << Message << "\n"; + return false; +} + +class DriverDiagnostics : public clang::DiagnosticConsumer { + void HandleDiagnostic(clang::DiagnosticsEngine::Level Level, + const clang::Diagnostic &Info) override { + clang::DiagnosticConsumer::HandleDiagnostic(Level, Info); + llvm::SmallString<256> Message; + Info.FormatDiagnostic(Message); + llvm::errs() << "cpsc: " << (Level >= clang::DiagnosticsEngine::Error + ? "error: " : "warning: ") << Message << "\n"; + } +}; + +} // namespace + +bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath) { + TemporaryDirectory Runtime; + if (auto EC = llvm::sys::fs::createUniqueDirectory("cpsc-native", Runtime.Path)) + return fail("cannot create native compilation directory: " + EC.message()); + + auto Manifest = llvm::json::parse(NativeRuntimeManifest); + if (!Manifest) return fail(llvm::toString(Manifest.takeError())); + auto &Metadata = *Manifest->getAsObject(); + llvm::SmallVector<uint8_t, 0> Data; + if (auto Error = llvm::compression::zlib::decompress( + llvm::ArrayRef(reinterpret_cast<const uint8_t *>(NativeRuntimeData), + sizeof(NativeRuntimeData) - 1), Data, NativeRuntimeSize)) + return fail("cannot unpack native link resources: " + llvm::toString(std::move(Error))); + + auto relocate = [&](llvm::StringRef Text) { + std::string Result = Text.str(); + const std::string Marker = "@CPSC_RUNTIME@"; + size_t Position = 0; + while ((Position = Result.find(Marker, Position)) != std::string::npos) { + Result.replace(Position, Marker.size(), Runtime.Path.str()); + Position += Runtime.Path.size(); + } + return Result; + }; + for (const auto &Entry : *Metadata.getArray("files")) { + const auto &File = *Entry.getAsObject(); + llvm::SmallString<256> Path(Runtime.Path); + llvm::sys::path::append(Path, *File.getString("path")); + if (auto EC = llvm::sys::fs::create_directories(llvm::sys::path::parent_path(Path))) + return fail("cannot prepare native link resources: " + EC.message()); + std::error_code EC; + llvm::raw_fd_ostream Output(Path, EC); + if (EC) return fail("cannot open native link resource: " + EC.message()); + llvm::StringRef Contents(reinterpret_cast<const char *>(Data.data()) + + *File.getInteger("offset"), *File.getInteger("size")); + if (*File.getBoolean("script")) Output << relocate(Contents); + else Output << Contents; + Output.close(); + if (Output.has_error()) { + auto Error = Output.error(); + Output.clear_error(); + return fail("cannot write native link resource: " + Error.message()); + } + } + + if (llvm::InitializeNativeTarget() || llvm::InitializeNativeTargetAsmPrinter()) + return fail("cannot initialize the native LLVM target"); + auto &Registry = *llvm::PassRegistry::getPassRegistry(); + llvm::initializeCore(Registry); + llvm::initializeCodeGen(Registry); + llvm::initializeScalarOpts(Registry); + llvm::initializeVectorization(Registry); + llvm::initializeTransformUtils(Registry); + llvm::initializeInstCombine(Registry); + llvm::initializeTarget(Registry); + llvm::Triple Triple(*Metadata.getString("target")); + std::string Error; + const auto *Target = llvm::TargetRegistry::lookupTarget(Triple, Error); + if (!Target) return fail(Error); + std::unique_ptr<llvm::TargetMachine> Machine(Target->createTargetMachine( + Triple, "generic", "", llvm::TargetOptions(), llvm::Reloc::PIC_)); + if (!Machine) return fail("cannot create the native LLVM target machine"); + auto Module = llvm::CloneModule(CG.getModule()); + Module->setTargetTriple(Triple); + Module->setDataLayout(Machine->createDataLayout()); + llvm::SmallString<256> ObjectPath(Runtime.Path); + llvm::sys::path::append(ObjectPath, "program.o"); + std::error_code EC; + llvm::raw_fd_ostream ObjectOutput(ObjectPath, EC); + if (EC) return fail("cannot open native object file: " + EC.message()); + llvm::legacy::PassManager Passes; + if (Machine->addPassesToEmitFile(Passes, ObjectOutput, nullptr, + llvm::CodeGenFileType::ObjectFile)) + return fail("native target cannot emit object files"); + Passes.run(*Module); + ObjectOutput.close(); + if (ObjectOutput.has_error()) { + auto Error = ObjectOutput.error(); + ObjectOutput.clear_error(); + return fail("cannot write native object file: " + Error.message()); + } + + // Link beside the destination so the final rename stays on one filesystem. + llvm::SmallString<256> StagedPath; + int FD; + if (auto EC = llvm::sys::fs::createUniqueFile(OutputPath + ".tmp-%%%%%%", FD, StagedPath)) + return fail("cannot open native output file: " + EC.message()); + llvm::FileRemover StagedOutput(StagedPath); + llvm::raw_fd_ostream CloseFD(FD, true); + CloseFD.close(); + + // All link inputs are explicit. Clang's driver provides job validation and + // platform command construction; no compiler subprocess is executed. + clang::DiagnosticOptions Options; + DriverDiagnostics Printer; + clang::DiagnosticsEngine Diagnostics(new clang::DiagnosticIDs, Options, &Printer, false); + clang::driver::Driver Driver("cpsc", Triple.str(), Diagnostics); + // The standard linker flavor constructs a job without checking for an + // installed linker. Its platform arguments are passed to embedded LLD. + std::vector<std::string> Args = {"cpsc", "--driver-mode=g++", "-nostdlib", "-nostdlib++", + "-fuse-ld=ld", "-o", std::string(StagedPath), std::string(ObjectPath)}; + for (const auto &Arg : *Metadata.getArray("arguments")) { + Args.push_back("-Xlinker"); + Args.push_back(relocate(*Arg.getAsString())); + } + std::vector<const char *> Argv; + for (const auto &Arg : Args) Argv.push_back(Arg.c_str()); + Driver.setCheckInputsExist(false); + std::unique_ptr<clang::driver::Compilation> Compilation(Driver.BuildCompilation(Argv)); + if (!Compilation || Diagnostics.hasErrorOccurred()) return false; + if (Compilation->getJobs().size() != 1) + return fail("expected one embedded linker job"); + const auto &Job = *Compilation->getJobs().begin(); + llvm::SmallVector<const char *, 64> LinkArgs; +#ifdef _WIN32 + LinkArgs.push_back("ld.lld"); + const lld::DriverDef Linkers[] = {{lld::MinGW, &lld::mingw::link}, {lld::WinLink, &lld::coff::link}}; +#elif defined(__APPLE__) + LinkArgs.push_back("ld64.lld"); + const lld::DriverDef Linkers[] = {{lld::Darwin, &lld::macho::link}}; +#else + LinkArgs.push_back("ld.lld"); + const lld::DriverDef Linkers[] = {{lld::Gnu, &lld::elf::link}}; +#endif + LinkArgs.append(Job.getArguments().begin(), Job.getArguments().end()); + auto Result = lld::lldMain(LinkArgs, llvm::outs(), llvm::errs(), Linkers); + if (!Result.canRunAgain) std::exit(1); + if (Result.retCode != 0) return fail("native linking failed"); + if (auto EC = llvm::sys::fs::setPermissions(StagedPath, + llvm::sys::fs::all_read | llvm::sys::fs::owner_write | llvm::sys::fs::all_exe)) + return fail("cannot set executable permissions: " + EC.message()); + if (auto EC = llvm::sys::fs::rename(StagedPath, OutputPath)) + return fail("cannot save native output: " + EC.message()); + return true; +} diff --git a/tools/driver/NativeCompiler.h b/tools/driver/NativeCompiler.h new file mode 100644 index 0000000..17933e9 --- /dev/null +++ b/tools/driver/NativeCompiler.h @@ -0,0 +1,9 @@ +#pragma once + +#include "llvm/ADT/StringRef.h" + +namespace cps { class CodeGen; } + +// Generate a host executable without an external compiler, linker or SDK. +// Failure leaves any existing output intact and reports a diagnostic. +bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath); diff --git a/tools/driver/prepare-native-runtime.py b/tools/driver/prepare-native-runtime.py new file mode 100644 index 0000000..8463eba --- /dev/null +++ b/tools/driver/prepare-native-runtime.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Freeze the build toolchain's native link inputs into the compiler.""" + +import argparse +import json +from pathlib import Path +import re +import shlex +import subprocess +import zlib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--platform", choices=("linux", "windows", "macos"), required=True) + parser.add_argument("--flag", action="append", default=[]) + args = parser.parse_args() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + + def compiler(*options): + return subprocess.run([args.compiler, *args.flag, *options], check=True, + capture_output=True, text=True) + + # An actual object lets the driver validate its inputs without linking a probe. + source = output.parent / "native-probe.cc" + obj = output.parent / "native-probe.o" + binary = output.parent / "native-probe.out" + source.write_text("int main() { return 0; }\n") + compiler("-c", str(source), "-o", str(obj)) + static = [] if args.platform == "macos" else ( + ["-static"] if args.platform == "windows" else + ["-static-libstdc++", "-static-libgcc"]) + trace = compiler("-###", "-fuse-ld=lld", *static, str(obj), "-o", str(binary)) + commands = [shlex.split(line.strip()) for line in trace.stderr.splitlines() + if line.lstrip().startswith('"')] + if len(commands) != 1: + raise RuntimeError("Expected one native linker command: " + trace.stderr) + command = commands[0][1:] + directories = [Path(arg[2:]) for arg in command if arg.startswith("-L")] + target = compiler("-dumpmachine").stdout.strip() + files = {} + scripts = set() + arguments = [] + root = "@CPSC_RUNTIME@" + + def resolve(name, static_only=False): + if name.startswith(":"): + names = [name[1:]] + elif args.platform == "macos": + names = ["lib" + name + suffix for suffix in (".tbd", ".dylib", ".a")] + elif static_only: + names = ["lib" + name + ".a"] + else: + names = ["lib" + name + suffix for suffix in (".dll.a", ".so", ".a")] + for directory in directories: + for filename in names: + path = directory / filename + if path.is_file(): + return path + for filename in names: + found = Path(compiler("-print-file-name=" + filename).stdout.strip()) + if found.is_file(): + return found + raise RuntimeError("Cannot find native link input " + name) + + sdk = None + if args.platform == "macos": + if "-syslibroot" not in command: + raise RuntimeError("A macOS SDK is required to prepare native linking") + sdk = Path(command[command.index("-syslibroot") + 1]) + directories += [sdk / "usr/lib"] + # System dylibs live in the dyld cache on users' machines. Embed their + # SDK link interfaces, not runtime dylibs or SDK headers. + for path in sorted((sdk / "usr/lib").rglob("*.tbd")): + files["sdk/" + path.relative_to(sdk).as_posix()] = path.read_bytes() + + mapped = {} + + def resource(path): + path = path.resolve() + if path in mapped: + return root + "/" + mapped[path] + if sdk and path.is_relative_to(sdk.resolve()): + name = "sdk/" + path.relative_to(sdk.resolve()).as_posix() + else: + name = "files/" + str(len(mapped)) + "/" + path.name + mapped[path] = name + data = path.read_bytes() + if b"GNU ld script" in data[:256]: + text = re.sub(r"/\*.*?\*/", "", data.decode(), flags=re.S) + # GNU libc/libm linker scripts contain absolute paths and nested + # GROUP/AS_NEEDED inputs. Relocate every referenced link input. + pattern = r"(?:/[^\s()]+|lib[\w.+-]+\.(?:so(?:\.[\d]+)*|a))" + + def relocate(match): + token = match.group() + dependency = Path(token) + if not dependency.is_file(): + dependency = resolve(":" + token) + return resource(dependency) + + data = re.sub(pattern, relocate, text).encode() + scripts.add(name) + files[name] = data + return root + "/" + name + + static_only = args.platform == "windows" + i = 0 + while i < len(command): + arg = command[i] + i += 1 + if arg == "-o": + i += 1 + elif arg == str(obj): + pass + elif arg == "-L": + i += 1 + elif arg.startswith("-L"): + pass + elif arg in ("-dynamic-linker", "--dynamic-linker"): + # The interpreter must remain an operating-system path. + arguments += [arg, command[i]] + i += 1 + elif arg == "-syslibroot": + arguments += [arg, root + "/sdk"] + i += 1 + elif arg in ("-Bstatic", "-Bdynamic"): + static_only = arg == "-Bstatic" + arguments.append(arg) + elif arg.startswith("-l"): + arguments.append(resource(resolve(arg[2:], static_only))) + elif Path(arg).is_file(): + arguments.append(resource(Path(arg))) + else: + arguments.append(arg) + + blob = bytearray() + manifest = {"target": target, "arguments": arguments, "files": []} + for name, data in files.items(): + manifest["files"].append({"path": name, "offset": len(blob), + "size": len(data), "script": name in scripts}) + blob.extend(data) + compressed = zlib.compress(blob, 9) + # Small separate literals avoid MSVC's individual string literal limit. + literals = "\n".join('"' + "".join(f"\\x{b:02x}" for b in compressed[i:i+1024]) + '"' + for i in range(0, len(compressed), 1024)) + content = ("// Generated native link resources; do not edit.\n" + f"static constexpr size_t NativeRuntimeSize = {len(blob)};\n" + "static const char NativeRuntimeData[] =\n" + literals + ";\n" + 'static const char NativeRuntimeManifest[] = R"CPSC(\n' + + json.dumps(manifest) + '\n)CPSC";\n') + if not output.exists() or output.read_text() != content: + output.write_text(content) + # Track the actual toolchain inputs, not only the generator script. + dependencies = [str(path) for path in mapped] + if sdk: + dependencies += [str(path) for path in (sdk / "usr/lib").rglob("*.tbd")] + (output.parent / "native-runtime-inputs.cmake").write_text( + "set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS\n" + + "\n".join(' [==[' + path + ']==]' for path in dependencies) + ")\n") + print(f"Embedded {len(files)} native link inputs ({len(compressed)} compressed bytes)") + + +if __name__ == "__main__": + main() |
