feat(driver): link INCLUDE_OBJ objects and check EXTERN symbols

Each EXTERN symbol must be defined by an INCLUDE_OBJ object file or by
the embedded C library selected with INCLUDE_OBJ "__libc". Objects are
linked natively, loaded by the embedded JIT and passed to lli.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

5 files changed, 221 insertions, 15 deletions

diff --git a/tools/driver/EmbeddedRunner.cc b/tools/driver/EmbeddedRunner.cc
index 8bfeccb..3216882 100644
--- a/tools/driver/EmbeddedRunner.cc
+++ b/tools/driver/EmbeddedRunner.cc
@@ -5,6 +5,7 @@
 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/TargetSelect.h"
 #include <cstdio>
 #include <cstdlib>
@@ -344,7 +345,7 @@ static llvm::Error defineRuntimeSymbols(llvm::orc::LLJIT &J) {
         llvm::orc::absoluteSymbols(std::move(Symbols)));
 }
 
-int runEmbedded(const cps::CodeGen &CG) {
+int runEmbedded(const cps::CodeGen &CG, const std::vector<std::string> &Objects) {
     if (llvm::InitializeNativeTarget() || llvm::InitializeNativeTargetAsmPrinter()) {
         llvm::errs() << "error: cannot initialize the embedded LLVM target\n";
         return 1;
@@ -405,6 +406,19 @@ int runEmbedded(const cps::CodeGen &CG) {
                      << llvm::toString(std::move(Err)) << "\n";
         return 1;
     }
+    for (const auto &Path : Objects) {
+        auto Buffer = llvm::MemoryBuffer::getFile(Path);
+        if (!Buffer) {
+            llvm::errs() << "error: cannot read object file '" << Path << "': "
+                         << Buffer.getError().message() << "\n";
+            return 1;
+        }
+        if (auto Err = J->addObjectFile(std::move(*Buffer))) {
+            llvm::errs() << "error: cannot load object file '" << Path << "': "
+                         << llvm::toString(std::move(Err)) << "\n";
+            return 1;
+        }
+    }
     if (auto Err = J->initialize(J->getMainJITDylib())) {
         llvm::errs() << "error: embedded LLVM execution failed: "
                      << llvm::toString(std::move(Err)) << "\n";
diff --git a/tools/driver/EmbeddedRunner.h b/tools/driver/EmbeddedRunner.h
index 1651f4d..adb7d58 100644
--- a/tools/driver/EmbeddedRunner.h
+++ b/tools/driver/EmbeddedRunner.h
@@ -2,4 +2,8 @@
 
 namespace cps { class CodeGen; }
 
-int runEmbedded(const cps::CodeGen &CG);
+#include <string>
+#include <vector>
+
+// Objects named by INCLUDE_OBJ are loaded into the JIT beside the program.
+int runEmbedded(const cps::CodeGen &CG, const std::vector<std::string> &Objects);
diff --git a/tools/driver/NativeCompiler.cc b/tools/driver/NativeCompiler.cc
index 8d798d7..4547203 100644
--- a/tools/driver/NativeCompiler.cc
+++ b/tools/driver/NativeCompiler.cc
@@ -10,6 +10,11 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/MC/TargetRegistry.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/TapiFile.h"
+#include "llvm/Object/TapiUniversal.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
@@ -58,21 +63,134 @@ class DriverDiagnostics : public clang::DiagnosticConsumer {
     }
 };
 
+// Mach-O and TAPI symbols carry the C global prefix '_'.
+void addSymbols(const llvm::object::SymbolicFile &File, bool Prefixed, llvm::StringSet<> &Out) {
+    auto Add = [&](const llvm::object::BasicSymbolRef &Symbol) {
+        auto Flags = Symbol.getFlags();
+        if (!Flags) {
+            llvm::consumeError(Flags.takeError());
+            return;
+        }
+        using llvm::object::BasicSymbolRef;
+        if (!(*Flags & BasicSymbolRef::SF_Global) || (*Flags & BasicSymbolRef::SF_Undefined)) return;
+        std::string Name;
+        llvm::raw_string_ostream Stream(Name);
+        if (auto Error = Symbol.printName(Stream)) {
+            llvm::consumeError(std::move(Error));
+            return;
+        }
+        llvm::StringRef Text(Name);
+        if (Prefixed && !Text.consume_front("_")) return;
+        Out.insert(Text);
+    };
+    for (const auto &Symbol : File.symbols()) Add(Symbol);
+    if (const auto *ELF = llvm::dyn_cast<llvm::object::ELFObjectFileBase>(&File))
+        for (const auto &Symbol : ELF->getDynamicSymbolIterators()) Add(Symbol);
+}
+
+// Objects, archives (recursively) and TAPI stubs; other inputs such as GNU
+// linker scripts contribute nothing.
+void addBinarySymbols(llvm::MemoryBufferRef Buffer, llvm::StringSet<> &Out) {
+    // Embedded resources and archive members need not be aligned for parsing.
+    auto Copy = llvm::MemoryBuffer::getMemBufferCopy(Buffer.getBuffer(), Buffer.getBufferIdentifier());
+    auto Binary = llvm::object::createBinary(Copy->getMemBufferRef());
+    if (!Binary) {
+        llvm::consumeError(Binary.takeError());
+        return;
+    }
+    if (auto *Archive = llvm::dyn_cast<llvm::object::Archive>(Binary->get())) {
+        llvm::Error Error = llvm::Error::success();
+        for (const auto &Child : Archive->children(Error)) {
+            auto Member = Child.getMemoryBufferRef();
+            if (Member) addBinarySymbols(*Member, Out);
+            else llvm::consumeError(Member.takeError());
+        }
+        llvm::consumeError(std::move(Error));
+    } else if (auto *Tapi = llvm::dyn_cast<llvm::object::TapiUniversal>(Binary->get())) {
+        for (const auto &Object : Tapi->objects()) {
+            auto File = Object.getAsObjectFile();
+            if (File) addSymbols(**File, true, Out);
+            else llvm::consumeError(File.takeError());
+        }
+    } else if (auto *File = llvm::dyn_cast<llvm::object::SymbolicFile>(Binary->get())) {
+        addSymbols(*File, File->isMachO(), Out);
+    }
+}
+
+bool isLibcInput(llvm::StringRef Path) {
+    llvm::StringRef Name = llvm::sys::path::filename(Path);
+#ifdef _WIN32
+    return Name.starts_with("libmsvcrt") || Name.starts_with("libucrt") ||
+           Name == "libmingwex.a" || Name == "libm.a";
+#elif defined(__APPLE__)
+    return Name == "libSystem.B.tbd" || Name == "libsystem_c.tbd" || Name == "libsystem_m.tbd";
+#else
+    return Name.starts_with("libc.so") || Name == "libc_nonshared.a" ||
+           Name.starts_with("libm.so") || Name.starts_with("libmvec.so");
+#endif
+}
+
+bool unpackRuntime(llvm::json::Object *&Metadata, std::optional<llvm::json::Value> &Manifest,
+                   llvm::SmallVector<uint8_t, 0> &Data) {
+    auto Parsed = llvm::json::parse(NativeRuntimeManifest);
+    if (!Parsed) return fail(llvm::toString(Parsed.takeError()));
+    Manifest = std::move(*Parsed);
+    Metadata = Manifest->getAsObject();
+    if (auto Error = llvm::compression::zlib::decompress(
+            llvm::ArrayRef(reinterpret_cast<const uint8_t *>(NativeRuntimeData),
+                           sizeof(NativeRuntimeData) - 1), Data, NativeRuntimeSize))
+        return fail("cannot unpack native link resources: " + llvm::toString(std::move(Error)));
+    return true;
+}
+
 } // namespace
 
-bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath, const char *Argv0) {
+bool collectObjectSymbols(llvm::StringRef Path, llvm::StringSet<> &Out, std::string &Error) {
+    auto Buffer = llvm::MemoryBuffer::getFile(Path);
+    if (!Buffer) {
+        Error = Buffer.getError().message();
+        return false;
+    }
+    auto Binary = llvm::object::createBinary((*Buffer)->getMemBufferRef());
+    if (!Binary) {
+        Error = llvm::toString(Binary.takeError());
+        return false;
+    }
+    auto *Object = llvm::dyn_cast<llvm::object::ObjectFile>(Binary->get());
+    if (!Object) {
+        Error = "not an object file";
+        return false;
+    }
+    addSymbols(*Object, Object->isMachO(), Out);
+    return true;
+}
+
+bool collectLibcSymbols(llvm::StringSet<> &Out) {
+    llvm::json::Object *Metadata = nullptr;
+    std::optional<llvm::json::Value> Manifest;
+    llvm::SmallVector<uint8_t, 0> Data;
+    if (!unpackRuntime(Metadata, Manifest, Data)) return false;
+    for (const auto &Entry : *Metadata->getArray("files")) {
+        const auto &File = *Entry.getAsObject();
+        if (*File.getBoolean("script") || !isLibcInput(*File.getString("path"))) continue;
+        llvm::StringRef Contents(reinterpret_cast<const char *>(Data.data()) +
+                                     *File.getInteger("offset"), *File.getInteger("size"));
+        addBinarySymbols(llvm::MemoryBufferRef(Contents, *File.getString("path")), Out);
+    }
+    return true;
+}
+
+bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath, const char *Argv0,
+                   const std::vector<std::string> &Objects) {
     TemporaryDirectory Runtime;
     if (auto EC = llvm::sys::fs::createUniqueDirectory("cpsc-native", Runtime.Path))
         return fail("cannot create native compilation directory: " + EC.message());
 
-    auto Manifest = llvm::json::parse(NativeRuntimeManifest);
-    if (!Manifest) return fail(llvm::toString(Manifest.takeError()));
-    auto &Metadata = *Manifest->getAsObject();
+    llvm::json::Object *MetadataObject = nullptr;
+    std::optional<llvm::json::Value> Manifest;
     llvm::SmallVector<uint8_t, 0> Data;
-    if (auto Error = llvm::compression::zlib::decompress(
-            llvm::ArrayRef(reinterpret_cast<const uint8_t *>(NativeRuntimeData),
-                           sizeof(NativeRuntimeData) - 1), Data, NativeRuntimeSize))
-        return fail("cannot unpack native link resources: " + llvm::toString(std::move(Error)));
+    if (!unpackRuntime(MetadataObject, Manifest, Data)) return false;
+    auto &Metadata = *MetadataObject;
 
     auto relocate = [&](llvm::StringRef Text) {
         std::string Result = Text.str();
@@ -161,6 +279,7 @@ bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath, const cha
     // cpsc's own path; only the job's arguments are passed to embedded LLD.
     std::vector<std::string> Args = {"cpsc", "--driver-mode=g++", "-nostdlib", "-nostdlib++",
         "-fuse-ld=lld", "-o", std::string(StagedPath), std::string(ObjectPath)};
+    Args.insert(Args.end(), Objects.begin(), Objects.end());
     Args.push_back("--ld-path=" + llvm::sys::fs::getMainExecutable(
         Argv0, reinterpret_cast<void *>(&compileNative)));
     for (const auto &Arg : *Metadata.getArray("arguments")) {
diff --git a/tools/driver/NativeCompiler.h b/tools/driver/NativeCompiler.h
index 5e13c4d..ffc38f1 100644
--- a/tools/driver/NativeCompiler.h
+++ b/tools/driver/NativeCompiler.h
@@ -1,9 +1,21 @@
 #pragma once
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+#include <string>
+#include <vector>
 
 namespace cps { class CodeGen; }
 
 // Generate a host executable without an external compiler, linker or SDK.
 // Failure leaves any existing output intact and reports a diagnostic.
-bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath, const char *Argv0);
+// Objects named by INCLUDE_OBJ are linked after the program object.
+bool compileNative(const cps::CodeGen &CG, llvm::StringRef OutputPath, const char *Argv0,
+                   const std::vector<std::string> &Objects);
+
+// Defined global C symbols of an INCLUDE_OBJ object file. Returns false with a
+// message when the file cannot be read or is not an object file.
+bool collectObjectSymbols(llvm::StringRef Path, llvm::StringSet<> &Out, std::string &Error);
+
+// Defined global symbols of the embedded C library (INCLUDE_OBJ "__libc").
+bool collectLibcSymbols(llvm::StringSet<> &Out);
diff --git a/tools/driver/main.cc b/tools/driver/main.cc
index 2908a0a..3ee221b 100644
--- a/tools/driver/main.cc
+++ b/tools/driver/main.cc
@@ -7,6 +7,7 @@
 #include "EmbeddedRunner.h"
 #endif
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #ifdef CPSC_RUN_PATH
@@ -194,6 +195,58 @@ int main(int argc, char **argv) {
                 Units[I].Visible[J] = Units[I].Visible[J] ||
                     (Units[I].Visible[K] && Units[K].Visible[J]);
 
+    // INCLUDE_OBJ is global: every input's objects join one link, and each
+    // EXTERN symbol must be defined by one of them ("__libc" is built in).
+    std::vector<std::string> Objects;
+    llvm::StringSet<> ExternSymbols;
+    bool LibcLoaded = false;
+    for (size_t I = 0; I < Units.size(); ++I) {
+        cps::DiagnosticSource = Units.size() > 1 ? Units[I].Path : "";
+        for (const auto &Stmt : Units[I].Statements) {
+            auto *Object = dynamic_cast<cps::IncludeObjStmtAST*>(Stmt.get());
+            if (!Object) continue;
+            if (Object->getPath() == "__libc") {
+                if (!LibcLoaded && !collectLibcSymbols(ExternSymbols)) return compilationTerminated();
+                LibcLoaded = true;
+                continue;
+            }
+            llvm::SmallString<256> Path(Object->getPath());
+            if (!llvm::sys::path::is_absolute(Path)) {
+                Path = llvm::sys::path::parent_path(Units[I].Path);
+                llvm::sys::path::append(Path, Object->getPath());
+            }
+            if (llvm::sys::path::extension(Path) != ".o") {
+                cps::sourceDiagnostic(Stmt->getLine(),
+                    "error: INCLUDE_OBJ '%s' must name a .o object file or \"__libc\" at line %d\n",
+                    Object->getPath().c_str(), Stmt->getLine());
+                return compilationTerminated();
+            }
+            bool Seen = false;
+            for (const auto &Existing : Objects)
+                Seen = Seen || llvm::sys::fs::equivalent(Path, Existing);
+            if (Seen) continue;
+            std::string Error;
+            if (!collectObjectSymbols(Path, ExternSymbols, Error)) {
+                cps::sourceDiagnostic(Stmt->getLine(), "error: cannot INCLUDE_OBJ '%s': %s at line %d\n",
+                    Object->getPath().c_str(), Error.c_str(), Stmt->getLine());
+                return compilationTerminated();
+            }
+            Objects.push_back(std::string(Path));
+        }
+    }
+    bool MissingSymbol = false;
+    for (size_t I = 0; I < Units.size(); ++I) {
+        cps::DiagnosticSource = Units.size() > 1 ? Units[I].Path : "";
+        for (const auto &Stmt : Units[I].Statements) {
+            auto *Extern = dynamic_cast<cps::ExternDeclAST*>(Stmt.get());
+            if (!Extern || ExternSymbols.contains(Extern->getProto()->getName())) continue;
+            cps::sourceDiagnostic(Stmt->getLine(), "error: C symbol '%s' not found in any INCLUDE_OBJ at line %d\n",
+                Extern->getProto()->getName().c_str(), Stmt->getLine());
+            MissingSymbol = true;
+        }
+    }
+    if (MissingSymbol) return compilationTerminated();
+
     cps::CodeGen CG;
     CG.compile(Units);
     if (CG.hadError()) return compilationTerminated();
@@ -216,12 +269,12 @@ int main(int argc, char **argv) {
     }
 
     if (Save && !EmitIR) {
-        if (!compileNative(CG, OutputPath, argv[0])) return compilationTerminated();
+        if (!compileNative(CG, OutputPath, argv[0], Objects)) return compilationTerminated();
         if (!Run) return 0;
     }
     std::string IRPath = Save && EmitIR ? OutputPath : std::string();
 #ifdef CPSC_RUN_EMBEDDED
-    if (Run && IRPath.empty()) return runEmbedded(CG);
+    if (Run && IRPath.empty()) return runEmbedded(CG, Objects);
 #endif
     std::error_code EC;
     int OutputFD;
@@ -259,7 +312,11 @@ int main(int argc, char **argv) {
         return 1;
     }
     std::string ExecutionError;
-    int Result = llvm::sys::ExecuteAndWait(LliPath, {LliPath, RunPath},
+    std::vector<std::string> LliArgs = {LliPath};
+    for (const auto &Object : Objects) LliArgs.push_back("-extra-object=" + Object);
+    LliArgs.push_back(std::string(RunPath));
+    std::vector<llvm::StringRef> LliArgv(LliArgs.begin(), LliArgs.end());
+    int Result = llvm::sys::ExecuteAndWait(LliPath, LliArgv,
                                          std::nullopt, {}, 0, 0, &ExecutionError);
     if (Result < 0) {
         llvm::errs() << "cpsc: error: lli execution failed (status " << Result << "): "
@@ -268,7 +325,7 @@ int main(int argc, char **argv) {
     }
     return Result;
 #elif defined(CPSC_RUN_EMBEDDED)
-    return runEmbedded(CG);
+    return runEmbedded(CG, Objects);
 #else
     return 0;
 #endif