feat(codegen): accept pointers to C-struct records in EXTERN signatures

A TYPE record whose fields are all C types, C structs or pointer types to
C structs now crosses the EXTERN "C" boundary through a pointer type.
Records by value are rejected for now, and a pointer to a record with a
non-C field names that field.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

2 files changed, 64 insertions, 2 deletions

diff --git a/lib/CodeGen/FunctionGen.cc b/lib/CodeGen/FunctionGen.cc
index 04f6a84..1cbf19f 100644
--- a/lib/CodeGen/FunctionGen.cc
+++ b/lib/CodeGen/FunctionGen.cc
@@ -298,6 +298,26 @@ bool FunctionGen::emitExternPrototype(PrototypeAST *Proto) {
         return Info->CSigned ? Attribute::SExt : Attribute::ZExt;
     };
 
+    // Records cross the C boundary only as C structs, for now through a
+    // pointer type. What names the parameter or return type in diagnostics.
+    auto CheckRecord = [&](const TypeInfo *Info, const std::string &What) {
+        const TypeInfo *Record = Info->Kind == TypeKind::Pointer
+            ? Types.resolve(Info->Pointer.PointeeTypeName) : Info;
+        if (!Record || !Record->isRecord() || Record->IsClass) return true;
+        std::string Why;
+        if (!Types.isCStruct(Record, &Why)) {
+            reportError("%s uses record %s, which is not a C struct: %s", What.c_str(),
+                        Record->Name.c_str(), Why.c_str());
+            return false;
+        }
+        if (Record == Info) {
+            reportError("%s cannot be record %s by value; use a pointer type to it", What.c_str(),
+                        Record->Name.c_str());
+            return false;
+        }
+        return true;
+    };
+
     FuncSig Sig;
     Sig.OwnerUnit = CurrentUnit;
     Sig.Name = Name;
@@ -317,7 +337,9 @@ bool FunctionGen::emitExternPrototype(PrototypeAST *Proto) {
             reportError("Unknown type %s", P.TypeName.c_str());
             return false;
         }
-        if (!Info->isC() || Info->isVoid()) {
+        if (!CheckRecord(Info, "EXTERN parameter " + std::to_string(I + 1) + " of " + Name))
+            return false;
+        if (!Types.isExternType(Info)) {
             reportError("EXTERN parameter %zu of %s must be a non-void C type (got %s)",
                         I + 1, Name.c_str(), P.TypeName.c_str());
             return false;
@@ -337,7 +359,8 @@ bool FunctionGen::emitExternPrototype(PrototypeAST *Proto) {
             reportError("Unknown type %s", Proto->getReturnType().c_str());
             return false;
         }
-        if (!Info->isC() || Info->isVoid()) {
+        if (!CheckRecord(Info, "The return type of EXTERN_FUNCTION " + Name)) return false;
+        if (!Types.isExternType(Info)) {
             reportError("EXTERN_FUNCTION %s must return a non-void C type (got %s)",
                         Name.c_str(), Proto->getReturnType().c_str());
             return false;
diff --git a/lib/CodeGen/TypeSystem.cc b/lib/CodeGen/TypeSystem.cc
index a84200b..c13d81e 100644
--- a/lib/CodeGen/TypeSystem.cc
+++ b/lib/CodeGen/TypeSystem.cc
@@ -1,6 +1,7 @@
 #include "cps/TypeSystem.h"
 #include "cps/TypeAST.h"
 #include "llvm/IR/Constants.h"
+#include <algorithm>
 #include <utility>
 
 using namespace llvm;
@@ -262,6 +263,44 @@ bool TypeSystem::definePointer(const std::string &Name,
     return true;
 }
 
+namespace {
+
+// Seen holds the records already being checked, so self-referential
+// pointer types (linked lists) end the recursion.
+bool checkCStruct(const TypeSystem &Types, const TypeInfo *Info, std::string *Why,
+                  std::vector<const TypeInfo *> &Seen) {
+    if (!Info || !Info->isRecord() || Info->IsClass) return false;
+    if (std::find(Seen.begin(), Seen.end(), Info) != Seen.end()) return true;
+    Seen.push_back(Info);
+    for (const auto &[FieldName, Field] : Info->Record.Fields) {
+        const TypeInfo *FieldInfo = Types.resolve(Field.TypeName);
+        if (FieldInfo && FieldInfo->isC()) continue;
+        const TypeInfo *Inner = FieldInfo && FieldInfo->Kind == TypeKind::Pointer
+            ? Types.resolve(FieldInfo->Pointer.PointeeTypeName) : FieldInfo;
+        if (Inner && Inner->isRecord() && !Inner->IsClass) {
+            if (checkCStruct(Types, Inner, Why, Seen)) continue;
+            return false;
+        }
+        if (Why) *Why = "field " + FieldName + " of " + Info->Name + " has type " + Field.TypeName;
+        return false;
+    }
+    return true;
+}
+
+} // namespace
+
+bool TypeSystem::isCStruct(const TypeInfo *Info, std::string *Why) const {
+    std::vector<const TypeInfo *> Seen;
+    return checkCStruct(*this, Info, Why, Seen);
+}
+
+bool TypeSystem::isExternType(const TypeInfo *Info) const {
+    if (!Info) return false;
+    if (Info->isC()) return !Info->isVoid();
+    if (Info->Kind == TypeKind::Pointer) return isCStruct(resolve(Info->Pointer.PointeeTypeName));
+    return isCStruct(Info);
+}
+
 const EnumConstant *TypeSystem::lookupEnumConstant(const std::string &Name) const {
     auto It = EnumConstants.find(Name);
     return It == EnumConstants.end() || (Visible && !Visible(It->second.TypeName))