diff options
| -rw-r--r-- | CMakeLists.txt | 26 | ||||
| -rw-r--r-- | cmake/EmbeddedLLVM.cmake | 2 | ||||
| -rw-r--r-- | cmake/NativeCompiler.cmake | 110 | ||||
| -rw-r--r-- | doc/build.md | 15 | ||||
| -rw-r--r-- | tools/driver/NativeCompiler.cc | 197 | ||||
| -rw-r--r-- | tools/driver/NativeCompiler.h | 9 | ||||
| -rw-r--r-- | tools/driver/prepare-native-runtime.py | 168 |
7 files changed, 507 insertions, 20 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 76b76a7..f875856 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,38 +65,40 @@ add_library(CPSCodeGen STATIC lib/CodeGen/FileHandler.cc ) -llvm_map_components_to_libnames(llvm_libs core support) +llvm_map_components_to_libnames(llvm_libs core support native passes) if(CPSC_RUN_MODE STREQUAL "EMBEDDED") - llvm_map_components_to_libnames(llvm_libs core support native mcjit) + llvm_map_components_to_libnames(llvm_libs core support native passes mcjit) foreach(llvm_lib IN LISTS llvm_libs) get_target_property(llvm_type ${llvm_lib} TYPE) if(NOT llvm_type STREQUAL "STATIC_LIBRARY") message(FATAL_ERROR "EMBEDDED requires static LLVM libraries: ${llvm_lib}") endif() endforeach() - include(cmake/EmbeddedLLVM.cmake) endif() +include(cmake/EmbeddedLLVM.cmake) add_executable(cpsc tools/driver/main.cc) target_compile_definitions(cpsc PRIVATE CPSC_RUN_${CPSC_RUN_MODE}) if(CPSC_RUN_MODE STREQUAL "EMBEDDED") target_sources(cpsc PRIVATE tools/driver/EmbeddedRunner.cc) - if(MINGW) - target_link_options(cpsc PRIVATE -static) - elseif(APPLE) - # LLVM's support interface can include unused Homebrew dylibs such as Z3. - target_link_options(cpsc PRIVATE -Wl,-dead_strip_dylibs) - elseif(UNIX AND NOT APPLE) - target_link_options(cpsc PRIVATE -static-libstdc++ -static-libgcc -Wl,--as-needed) - endif() endif() -target_link_libraries(cpsc +if(MINGW) + target_link_options(cpsc PRIVATE -static) +elseif(APPLE) + # LLVM's support interface can include unused Homebrew dylibs such as Z3. + target_link_options(cpsc PRIVATE -Wl,-dead_strip_dylibs) +elseif(UNIX AND NOT APPLE) + target_link_options(cpsc PRIVATE -static-libstdc++ -static-libgcc -Wl,--as-needed) +endif() +target_link_libraries(cpsc PRIVATE CPSCodeGen CPSParser CPSLexer ${llvm_libs} ) +include(cmake/NativeCompiler.cmake) + include(cmake/Manual.cmake) configure_file(cmake/Install.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/Install.cmake" @ONLY) diff --git a/cmake/EmbeddedLLVM.cmake b/cmake/EmbeddedLLVM.cmake index c7ad5d4..cb08edc 100644 --- a/cmake/EmbeddedLLVM.cmake +++ b/cmake/EmbeddedLLVM.cmake @@ -17,7 +17,7 @@ foreach(cpsc_dependency IN LISTS cpsc_support_libs) cpsc_find_library(CPSC_STATIC_${cpsc_library_name} ${cpsc_library_names}) if(NOT CPSC_STATIC_${cpsc_library_name}) message(FATAL_ERROR - "EMBEDDED requires a static archive for ${cpsc_dependency}; download or install its development package") + "cpsc requires a static archive for ${cpsc_dependency}; download or install its development package") endif() string(REPLACE "${cpsc_dependency}" "${CPSC_STATIC_${cpsc_library_name}}" cpsc_support_libs "${cpsc_support_libs}") diff --git a/cmake/NativeCompiler.cmake b/cmake/NativeCompiler.cmake new file mode 100644 index 0000000..8371ee2 --- /dev/null +++ b/cmake/NativeCompiler.cmake @@ -0,0 +1,110 @@ +# Native output is independent of the optional -r execution engine. +find_package(Clang REQUIRED CONFIG HINTS "${LLVM_INSTALL_PREFIX}/lib/cmake/clang") +cpsc_find_dependency(PATH LLD_DIR NAMES LLDConfig.cmake PATH_SUFFIXES lib/cmake/lld cmake) +find_package(LLD REQUIRED CONFIG HINTS "${LLVM_INSTALL_PREFIX}/lib/cmake/lld") +find_package(Python3 3.9 REQUIRED COMPONENTS Interpreter) + +if(MINGW) + set(cpsc_native_platform windows) + set(cpsc_linker_libraries lldMinGW lldCOFF) +elseif(APPLE) + set(cpsc_native_platform macos) + set(cpsc_linker_libraries lldMachO) +else() + set(cpsc_native_platform linux) + set(cpsc_linker_libraries lldELF) +endif() + +set(cpsc_native_flags) +get_filename_component(cpsc_lld_prefix "${LLD_DIR}/../../.." ABSOLUTE) +if(APPLE) + set(cpsc_linker_names ld64.lld) +else() + set(cpsc_linker_names ld.lld-23 ld.lld) +endif() +find_program(CPSC_NATIVE_LINKER NAMES ${cpsc_linker_names} + HINTS "${cpsc_lld_prefix}/bin" ${CPSC_LOCAL_BIN_DIRS}) +if(NOT CPSC_NATIVE_LINKER) + message(FATAL_ERROR "Install the LLD 23 executable to prepare native link resources") +endif() +list(APPEND cpsc_native_flags "--flag=--ld-path=${CPSC_NATIVE_LINKER}") +target_link_options(cpsc PRIVATE -fuse-ld=lld "--ld-path=${CPSC_NATIVE_LINKER}") +if(CMAKE_SYSROOT) + list(APPEND cpsc_native_flags "--flag=--sysroot=${CMAKE_SYSROOT}") +endif() +if(APPLE) + if(NOT CMAKE_OSX_SYSROOT) + execute_process(COMMAND xcrun --show-sdk-path + OUTPUT_VARIABLE CMAKE_OSX_SYSROOT OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE cpsc_sdk_result) + if(NOT cpsc_sdk_result EQUAL 0) + message(FATAL_ERROR "Cannot locate the macOS SDK") + endif() + endif() + list(APPEND cpsc_native_flags "--flag=-isysroot" "--flag=${CMAKE_OSX_SYSROOT}") + if(CMAKE_OSX_DEPLOYMENT_TARGET) + list(APPEND cpsc_native_flags "--flag=-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}") + endif() +endif() +set(cpsc_runtime_generator "${CMAKE_SOURCE_DIR}/tools/driver/prepare-native-runtime.py") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${cpsc_runtime_generator}") +execute_process(COMMAND "${Python3_EXECUTABLE}" "${cpsc_runtime_generator}" + "--compiler=${CMAKE_CXX_COMPILER}" + "--output=${CMAKE_BINARY_DIR}/native-runtime/NativeRuntime.inc" + "--platform=${cpsc_native_platform}" ${cpsc_native_flags} + RESULT_VARIABLE cpsc_runtime_result OUTPUT_VARIABLE cpsc_runtime_output + ERROR_VARIABLE cpsc_runtime_error) +if(NOT cpsc_runtime_result EQUAL 0) + message(FATAL_ERROR "Cannot prepare native link resources: ${cpsc_runtime_error}") +endif() +message(STATUS "${cpsc_runtime_output}") +include("${CMAKE_BINARY_DIR}/native-runtime/native-runtime-inputs.cmake") + +target_sources(cpsc PRIVATE tools/driver/NativeCompiler.cc) +target_include_directories(cpsc PRIVATE ${CLANG_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS} + "${CMAKE_BINARY_DIR}/native-runtime") +target_link_libraries(cpsc PRIVATE clangDriver ${cpsc_linker_libraries}) +# Distribution Clang/LLD archives can still name the shared LLVM umbrella +# library in their imported interfaces. Resolve those edges to static archives. +add_library(CPSCStaticLLVM INTERFACE) +foreach(cpsc_llvm_library IN LISTS LLVM_AVAILABLE_LIBS) + # Standalone tools are not driver/linker dependencies and may reference + # optional packages absent from binary SDK installations. + if(cpsc_llvm_library MATCHES "^LLVM(LineEditor|BOLT.*|Exegesis.*|OptDriver)$") + continue() + endif() + if(TARGET ${cpsc_llvm_library}) + get_target_property(cpsc_llvm_type ${cpsc_llvm_library} TYPE) + if(cpsc_llvm_type STREQUAL "STATIC_LIBRARY") + target_link_libraries(CPSCStaticLLVM INTERFACE ${cpsc_llvm_library}) + endif() + endif() +endforeach() +foreach(cpsc_import IN LISTS CLANG_EXPORTED_TARGETS LLD_EXPORTED_TARGETS LLVM_AVAILABLE_LIBS) + if(NOT TARGET ${cpsc_import}) + continue() + endif() + get_target_property(cpsc_interface ${cpsc_import} INTERFACE_LINK_LIBRARIES) + if(cpsc_interface) + list(FIND cpsc_interface LLVM cpsc_shared_llvm_index) + if(NOT cpsc_shared_llvm_index EQUAL -1) + list(REMOVE_ITEM cpsc_interface LLVM) + if(NOT cpsc_import IN_LIST LLVM_AVAILABLE_LIBS) + list(APPEND cpsc_interface CPSCStaticLLVM) + endif() + endif() + if(CPSC_STATIC_z) + list(TRANSFORM cpsc_interface REPLACE "^ZLIB::ZLIB$" "${CPSC_STATIC_z}") + endif() + if(CPSC_STATIC_zstd) + list(TRANSFORM cpsc_interface REPLACE "^zstd::libzstd.*$" "${CPSC_STATIC_zstd}") + endif() + set_property(TARGET ${cpsc_import} PROPERTY INTERFACE_LINK_LIBRARIES "${cpsc_interface}") + endif() +endforeach() +foreach(cpsc_native_library clangDriver ${cpsc_linker_libraries}) + get_target_property(cpsc_native_type ${cpsc_native_library} TYPE) + if(NOT cpsc_native_type STREQUAL "STATIC_LIBRARY") + message(FATAL_ERROR "Native output requires static ${cpsc_native_library}") + endif() +endforeach() diff --git a/doc/build.md b/doc/build.md index d686689..08b4f24 100644 --- a/doc/build.md +++ b/doc/build.md @@ -30,8 +30,7 @@ version. CMake rebuilds its extraction directory when the package set changes. If you use a different CMake build directory, download into its `deps/packages/` instead. Removing the build directory also removes its downloaded dependencies. -All three execution modes need LLVM development libraries to build, including -`NONE`. Users of an `EMBEDDED` executable do not need LLVM or `lli`. +All three execution modes need LLVM, Clang and LLD 23 development libraries, Python 3.9 or newer, and the native C/C++ development libraries to build, including `NONE`. Users of an `EMBEDDED` executable do not need LLVM or `lli`. CMake strictly requires LLVM development libraries and both C/C++ compilers to have major version 23. The compilers must be upstream Clang; GCC and AppleClang are rejected. Any 23.x release is accepted; 22 and 24 are rejected. LLVM/Clang @@ -46,7 +45,7 @@ Use a distribution repository that provides LLVM 23. Recommended build tools: ```bash sudo apt-get update -sudo apt-get install -y --no-install-recommends build-essential cmake clang-23 +sudo apt-get install -y --no-install-recommends build-essential cmake clang-23 lld-23 python3 ``` Recommended LLVM and support-package downloads: @@ -57,7 +56,7 @@ mkdir -p build/deps/packages cd build/deps/packages apt-get download \ llvm-23-dev llvm-23 llvm-23-runtime llvm-23-tools llvm-23-linker-tools \ - libllvm23 libclang-cpp23 libz3-dev libz3-4 \ + libllvm23 libclang-cpp23 libclang-23-dev liblld-23-dev libpolly-23-dev libz3-dev libz3-4 \ libffi-dev libffi8 libxml2-dev libxml2 \ zlib1g-dev zlib1g libzstd-dev libzstd1 libncurses-dev libtinfo6 ) @@ -77,7 +76,8 @@ pacman -Syu pacman -S --needed \ mingw-w64-x86_64-cmake \ mingw-w64-x86_64-make \ - mingw-w64-x86_64-clang + mingw-w64-x86_64-clang \ + mingw-w64-x86_64-python ``` Confirm that both installed compilers report major version 23: @@ -108,6 +108,7 @@ Recommended dependency downloads to the project build directory: ```bash mkdir -p build/deps/packages pacman -Sw --noconfirm --cachedir "$PWD/build/deps/packages" \ + mingw-w64-x86_64-clang \ mingw-w64-x86_64-llvm \ mingw-w64-x86_64-llvm-libs \ mingw-w64-x86_64-llvm-tools \ @@ -131,7 +132,7 @@ relocation tools with `xcode-select --install` if needed. The LLVM bottle below provides Clang 23. Install CMake with Homebrew: ```bash -brew install cmake +brew install cmake python lld ``` Recommended binary-bottle downloads to the build directory: @@ -139,7 +140,7 @@ Recommended binary-bottle downloads to the build directory: ```bash mkdir -p build/deps/packages HOMEBREW_CACHE="$PWD/build/deps/packages" brew fetch --force-bottle \ - llvm zlib zstd ncurses lz4 xz + llvm lld zlib zstd ncurses lz4 xz ``` [Homebrew's `llvm` formula](https://formulae.brew.sh/formula/llvm) currently diff --git a/tools/driver/NativeCompiler.cc b/tools/driver/NativeCompiler.cc new file mode 100644 index 0000000..f2ea3dc --- /dev/null +++ b/tools/driver/NativeCompiler.cc @@ -0,0 +1,197 @@ +#include "NativeCompiler.h" +#include "cps/CodeGen.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "lld/Common/Driver.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/InitializePasses.h" +#include "llvm/PassRegistry.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include <cstdlib> + +#ifdef _WIN32 +LLD_HAS_DRIVER(mingw) +LLD_HAS_DRIVER(coff) +#elif defined(__APPLE__) +LLD_HAS_DRIVER(macho) +#else +LLD_HAS_DRIVER(elf) +#endif + +namespace { + +#include "NativeRuntime.inc" + +struct TemporaryDirectory { + llvm::SmallString<256> Path; + ~TemporaryDirectory() { + if (!Path.empty()) llvm::sys::fs::remove_directories(Path); + } +}; + +bool fail(const llvm::Twine &Message) { + llvm::errs() << "cpsc: fatal error: " << Message << "\n"; + return false; +} + +class DriverDiagnostics : public clang::DiagnosticConsumer { + void HandleDiagnostic(clang::DiagnosticsEngine::Level Level, + const clang::Diagnostic &Info) override { + clang::DiagnosticConsumer::HandleDiagnostic(Level, Info); + llvm::SmallString<256> Message; + Info.FormatDiagnostic(Message); + llvm::errs() << "cpsc: " << (Level >= clang::DiagnosticsEngine::Error + ? "error: " : "warning: ") << Message << "\n"; + } +}; + +} // namespace + +bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath) { + TemporaryDirectory Runtime; + if (auto EC = llvm::sys::fs::createUniqueDirectory("cpsc-native", Runtime.Path)) + return fail("cannot create native compilation directory: " + EC.message()); + + auto Manifest = llvm::json::parse(NativeRuntimeManifest); + if (!Manifest) return fail(llvm::toString(Manifest.takeError())); + auto &Metadata = *Manifest->getAsObject(); + llvm::SmallVector<uint8_t, 0> Data; + if (auto Error = llvm::compression::zlib::decompress( + llvm::ArrayRef(reinterpret_cast<const uint8_t *>(NativeRuntimeData), + sizeof(NativeRuntimeData) - 1), Data, NativeRuntimeSize)) + return fail("cannot unpack native link resources: " + llvm::toString(std::move(Error))); + + auto relocate = [&](llvm::StringRef Text) { + std::string Result = Text.str(); + const std::string Marker = "@CPSC_RUNTIME@"; + size_t Position = 0; + while ((Position = Result.find(Marker, Position)) != std::string::npos) { + Result.replace(Position, Marker.size(), Runtime.Path.str()); + Position += Runtime.Path.size(); + } + return Result; + }; + for (const auto &Entry : *Metadata.getArray("files")) { + const auto &File = *Entry.getAsObject(); + llvm::SmallString<256> Path(Runtime.Path); + llvm::sys::path::append(Path, *File.getString("path")); + if (auto EC = llvm::sys::fs::create_directories(llvm::sys::path::parent_path(Path))) + return fail("cannot prepare native link resources: " + EC.message()); + std::error_code EC; + llvm::raw_fd_ostream Output(Path, EC); + if (EC) return fail("cannot open native link resource: " + EC.message()); + llvm::StringRef Contents(reinterpret_cast<const char *>(Data.data()) + + *File.getInteger("offset"), *File.getInteger("size")); + if (*File.getBoolean("script")) Output << relocate(Contents); + else Output << Contents; + Output.close(); + if (Output.has_error()) { + auto Error = Output.error(); + Output.clear_error(); + return fail("cannot write native link resource: " + Error.message()); + } + } + + if (llvm::InitializeNativeTarget() || llvm::InitializeNativeTargetAsmPrinter()) + return fail("cannot initialize the native LLVM target"); + auto &Registry = *llvm::PassRegistry::getPassRegistry(); + llvm::initializeCore(Registry); + llvm::initializeCodeGen(Registry); + llvm::initializeScalarOpts(Registry); + llvm::initializeVectorization(Registry); + llvm::initializeTransformUtils(Registry); + llvm::initializeInstCombine(Registry); + llvm::initializeTarget(Registry); + llvm::Triple Triple(*Metadata.getString("target")); + std::string Error; + const auto *Target = llvm::TargetRegistry::lookupTarget(Triple, Error); + if (!Target) return fail(Error); + std::unique_ptr<llvm::TargetMachine> Machine(Target->createTargetMachine( + Triple, "generic", "", llvm::TargetOptions(), llvm::Reloc::PIC_)); + if (!Machine) return fail("cannot create the native LLVM target machine"); + auto Module = llvm::CloneModule(CG.getModule()); + Module->setTargetTriple(Triple); + Module->setDataLayout(Machine->createDataLayout()); + llvm::SmallString<256> ObjectPath(Runtime.Path); + llvm::sys::path::append(ObjectPath, "program.o"); + std::error_code EC; + llvm::raw_fd_ostream ObjectOutput(ObjectPath, EC); + if (EC) return fail("cannot open native object file: " + EC.message()); + llvm::legacy::PassManager Passes; + if (Machine->addPassesToEmitFile(Passes, ObjectOutput, nullptr, + llvm::CodeGenFileType::ObjectFile)) + return fail("native target cannot emit object files"); + Passes.run(*Module); + ObjectOutput.close(); + if (ObjectOutput.has_error()) { + auto Error = ObjectOutput.error(); + ObjectOutput.clear_error(); + return fail("cannot write native object file: " + Error.message()); + } + + // Link beside the destination so the final rename stays on one filesystem. + llvm::SmallString<256> StagedPath; + int FD; + if (auto EC = llvm::sys::fs::createUniqueFile(OutputPath + ".tmp-%%%%%%", FD, StagedPath)) + return fail("cannot open native output file: " + EC.message()); + llvm::FileRemover StagedOutput(StagedPath); + llvm::raw_fd_ostream CloseFD(FD, true); + CloseFD.close(); + + // All link inputs are explicit. Clang's driver provides job validation and + // platform command construction; no compiler subprocess is executed. + clang::DiagnosticOptions Options; + DriverDiagnostics Printer; + clang::DiagnosticsEngine Diagnostics(new clang::DiagnosticIDs, Options, &Printer, false); + clang::driver::Driver Driver("cpsc", Triple.str(), Diagnostics); + // The standard linker flavor constructs a job without checking for an + // installed linker. Its platform arguments are passed to embedded LLD. + std::vector<std::string> Args = {"cpsc", "--driver-mode=g++", "-nostdlib", "-nostdlib++", + "-fuse-ld=ld", "-o", std::string(StagedPath), std::string(ObjectPath)}; + for (const auto &Arg : *Metadata.getArray("arguments")) { + Args.push_back("-Xlinker"); + Args.push_back(relocate(*Arg.getAsString())); + } + std::vector<const char *> Argv; + for (const auto &Arg : Args) Argv.push_back(Arg.c_str()); + Driver.setCheckInputsExist(false); + std::unique_ptr<clang::driver::Compilation> Compilation(Driver.BuildCompilation(Argv)); + if (!Compilation || Diagnostics.hasErrorOccurred()) return false; + if (Compilation->getJobs().size() != 1) + return fail("expected one embedded linker job"); + const auto &Job = *Compilation->getJobs().begin(); + llvm::SmallVector<const char *, 64> LinkArgs; +#ifdef _WIN32 + LinkArgs.push_back("ld.lld"); + const lld::DriverDef Linkers[] = {{lld::MinGW, &lld::mingw::link}, {lld::WinLink, &lld::coff::link}}; +#elif defined(__APPLE__) + LinkArgs.push_back("ld64.lld"); + const lld::DriverDef Linkers[] = {{lld::Darwin, &lld::macho::link}}; +#else + LinkArgs.push_back("ld.lld"); + const lld::DriverDef Linkers[] = {{lld::Gnu, &lld::elf::link}}; +#endif + LinkArgs.append(Job.getArguments().begin(), Job.getArguments().end()); + auto Result = lld::lldMain(LinkArgs, llvm::outs(), llvm::errs(), Linkers); + if (!Result.canRunAgain) std::exit(1); + if (Result.retCode != 0) return fail("native linking failed"); + if (auto EC = llvm::sys::fs::setPermissions(StagedPath, + llvm::sys::fs::all_read | llvm::sys::fs::owner_write | llvm::sys::fs::all_exe)) + return fail("cannot set executable permissions: " + EC.message()); + if (auto EC = llvm::sys::fs::rename(StagedPath, OutputPath)) + return fail("cannot save native output: " + EC.message()); + return true; +} diff --git a/tools/driver/NativeCompiler.h b/tools/driver/NativeCompiler.h new file mode 100644 index 0000000..17933e9 --- /dev/null +++ b/tools/driver/NativeCompiler.h @@ -0,0 +1,9 @@ +#pragma once + +#include "llvm/ADT/StringRef.h" + +namespace cps { class CodeGen; } + +// Generate a host executable without an external compiler, linker or SDK. +// Failure leaves any existing output intact and reports a diagnostic. +bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath); diff --git a/tools/driver/prepare-native-runtime.py b/tools/driver/prepare-native-runtime.py new file mode 100644 index 0000000..8463eba --- /dev/null +++ b/tools/driver/prepare-native-runtime.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Freeze the build toolchain's native link inputs into the compiler.""" + +import argparse +import json +from pathlib import Path +import re +import shlex +import subprocess +import zlib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--compiler", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--platform", choices=("linux", "windows", "macos"), required=True) + parser.add_argument("--flag", action="append", default=[]) + args = parser.parse_args() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + + def compiler(*options): + return subprocess.run([args.compiler, *args.flag, *options], check=True, + capture_output=True, text=True) + + # An actual object lets the driver validate its inputs without linking a probe. + source = output.parent / "native-probe.cc" + obj = output.parent / "native-probe.o" + binary = output.parent / "native-probe.out" + source.write_text("int main() { return 0; }\n") + compiler("-c", str(source), "-o", str(obj)) + static = [] if args.platform == "macos" else ( + ["-static"] if args.platform == "windows" else + ["-static-libstdc++", "-static-libgcc"]) + trace = compiler("-###", "-fuse-ld=lld", *static, str(obj), "-o", str(binary)) + commands = [shlex.split(line.strip()) for line in trace.stderr.splitlines() + if line.lstrip().startswith('"')] + if len(commands) != 1: + raise RuntimeError("Expected one native linker command: " + trace.stderr) + command = commands[0][1:] + directories = [Path(arg[2:]) for arg in command if arg.startswith("-L")] + target = compiler("-dumpmachine").stdout.strip() + files = {} + scripts = set() + arguments = [] + root = "@CPSC_RUNTIME@" + + def resolve(name, static_only=False): + if name.startswith(":"): + names = [name[1:]] + elif args.platform == "macos": + names = ["lib" + name + suffix for suffix in (".tbd", ".dylib", ".a")] + elif static_only: + names = ["lib" + name + ".a"] + else: + names = ["lib" + name + suffix for suffix in (".dll.a", ".so", ".a")] + for directory in directories: + for filename in names: + path = directory / filename + if path.is_file(): + return path + for filename in names: + found = Path(compiler("-print-file-name=" + filename).stdout.strip()) + if found.is_file(): + return found + raise RuntimeError("Cannot find native link input " + name) + + sdk = None + if args.platform == "macos": + if "-syslibroot" not in command: + raise RuntimeError("A macOS SDK is required to prepare native linking") + sdk = Path(command[command.index("-syslibroot") + 1]) + directories += [sdk / "usr/lib"] + # System dylibs live in the dyld cache on users' machines. Embed their + # SDK link interfaces, not runtime dylibs or SDK headers. + for path in sorted((sdk / "usr/lib").rglob("*.tbd")): + files["sdk/" + path.relative_to(sdk).as_posix()] = path.read_bytes() + + mapped = {} + + def resource(path): + path = path.resolve() + if path in mapped: + return root + "/" + mapped[path] + if sdk and path.is_relative_to(sdk.resolve()): + name = "sdk/" + path.relative_to(sdk.resolve()).as_posix() + else: + name = "files/" + str(len(mapped)) + "/" + path.name + mapped[path] = name + data = path.read_bytes() + if b"GNU ld script" in data[:256]: + text = re.sub(r"/\*.*?\*/", "", data.decode(), flags=re.S) + # GNU libc/libm linker scripts contain absolute paths and nested + # GROUP/AS_NEEDED inputs. Relocate every referenced link input. + pattern = r"(?:/[^\s()]+|lib[\w.+-]+\.(?:so(?:\.[\d]+)*|a))" + + def relocate(match): + token = match.group() + dependency = Path(token) + if not dependency.is_file(): + dependency = resolve(":" + token) + return resource(dependency) + + data = re.sub(pattern, relocate, text).encode() + scripts.add(name) + files[name] = data + return root + "/" + name + + static_only = args.platform == "windows" + i = 0 + while i < len(command): + arg = command[i] + i += 1 + if arg == "-o": + i += 1 + elif arg == str(obj): + pass + elif arg == "-L": + i += 1 + elif arg.startswith("-L"): + pass + elif arg in ("-dynamic-linker", "--dynamic-linker"): + # The interpreter must remain an operating-system path. + arguments += [arg, command[i]] + i += 1 + elif arg == "-syslibroot": + arguments += [arg, root + "/sdk"] + i += 1 + elif arg in ("-Bstatic", "-Bdynamic"): + static_only = arg == "-Bstatic" + arguments.append(arg) + elif arg.startswith("-l"): + arguments.append(resource(resolve(arg[2:], static_only))) + elif Path(arg).is_file(): + arguments.append(resource(Path(arg))) + else: + arguments.append(arg) + + blob = bytearray() + manifest = {"target": target, "arguments": arguments, "files": []} + for name, data in files.items(): + manifest["files"].append({"path": name, "offset": len(blob), + "size": len(data), "script": name in scripts}) + blob.extend(data) + compressed = zlib.compress(blob, 9) + # Small separate literals avoid MSVC's individual string literal limit. + literals = "\n".join('"' + "".join(f"\\x{b:02x}" for b in compressed[i:i+1024]) + '"' + for i in range(0, len(compressed), 1024)) + content = ("// Generated native link resources; do not edit.\n" + f"static constexpr size_t NativeRuntimeSize = {len(blob)};\n" + "static const char NativeRuntimeData[] =\n" + literals + ";\n" + 'static const char NativeRuntimeManifest[] = R"CPSC(\n' + + json.dumps(manifest) + '\n)CPSC";\n') + if not output.exists() or output.read_text() != content: + output.write_text(content) + # Track the actual toolchain inputs, not only the generator script. + dependencies = [str(path) for path in mapped] + if sdk: + dependencies += [str(path) for path in (sdk / "usr/lib").rglob("*.tbd")] + (output.parent / "native-runtime-inputs.cmake").write_text( + "set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS\n" + + "\n".join(' [==[' + path + ']==]' for path in dependencies) + ")\n") + print(f"Embedded {len(files)} native link inputs ({len(compressed)} compressed bytes)") + + +if __name__ == "__main__": + main() |