[JITLink][AArch32] Multi-stub support for armv7/thumbv7

Author: weliveindetail

llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h

@@ -318,64 +318,31 @@ inline Error applyFixup(LinkGraph &G, Block &B, const Edge &E,
   llvm_unreachable("Relocation must be of class Data, Arm or Thumb");
 }
 
-/// Stubs builder for v7 emits non-position-independent Thumb stubs.
-///
-/// Right now we only have one default stub kind, but we want to extend this
-/// and allow creation of specific kinds in the future (e.g. branch range
-/// extension or interworking).
-///
-/// Let's keep it simple for the moment and not wire this through a GOT.
-///
-class StubsManager_v7 : public TableManager<StubsManager_v7> {
+/// Stubs builder for v7 emits non-position-independent Arm and Thumb stubs.
+class StubsManager_v7 {
 public:
   StubsManager_v7() = default;
 
   /// Name of the object file section that will contain all our stubs.
   static StringRef getSectionName() {
-    return "__llvm_jitlink_aarch32_STUBS_Thumbv7";
+    return "__llvm_jitlink_aarch32_STUBS_v7";
   }
 
   /// Implements link-graph traversal via visitExistingEdges().
-  bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
-    if (E.getTarget().isDefined())
-      return false;
-
-    switch (E.getKind()) {
-    case Thumb_Call:
-    case Thumb_Jump24: {
-      DEBUG_WITH_TYPE("jitlink", {
-        dbgs() << "  Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
-               << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
-               << formatv("{0:x}", E.getOffset()) << ")\n";
-      });
-      E.setTarget(this->getEntryForTarget(G, E.getTarget()));
-      return true;
-    }
-    }
-    return false;
-  }
-
-  /// Create a branch range extension stub with Thumb encoding for v7 CPUs.
-  Symbol &createEntry(LinkGraph &G, Symbol &Target);
+  bool visitEdge(LinkGraph &G, Block *B, Edge &E);
 
 private:
-  /// Create a new node in the link-graph for the given stub template.
-  template <size_t Size>
-  Block &addStub(LinkGraph &G, const uint8_t (&Code)[Size],
-                 uint64_t Alignment) {
-    ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
-    return G.createContentBlock(getStubsSection(G), Template,
-                                orc::ExecutorAddr(), Alignment, 0);
-  }
-
-  /// Get or create the object file section that will contain all our stubs.
-  Section &getStubsSection(LinkGraph &G) {
-    if (!StubsSection)
-      StubsSection = &G.createSection(getSectionName(),
-                                      orc::MemProt::Read | orc::MemProt::Exec);
-    return *StubsSection;
+  // Two slots per external: Arm and Thumb
+  using StubMapEntry = std::tuple<Symbol *, Symbol *>;
+
+  Symbol *&getStubSymbolSlot(StringRef Name, bool Thumb) {
+    StubMapEntry &Stubs = StubMap.try_emplace(Name).first->second;
+    if (Thumb)
+      return std::get<1>(Stubs);
+    return std::get<0>(Stubs);
   }
 
+  DenseMap<StringRef, StubMapEntry> StubMap;
   Section *StubsSection = nullptr;
 };
 

llvm/lib/ExecutionEngine/JITLink/aarch32.cpp

@@ -15,6 +15,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ManagedStatic.h"
@@ -678,27 +679,127 @@ Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E,
   }
 }
 
+const uint8_t Armv7ABS[] = {
+    0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000     ; lower 16-bit
+    0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000     ; upper 16-bit
+    0x1c, 0xff, 0x2f, 0xe1  // bx   r12
+};
+
 const uint8_t Thumbv7ABS[] = {
     0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000    ; lower 16-bit
     0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000    ; upper 16-bit
     0x60, 0x47              // bx   r12
 };
 
-Symbol &StubsManager_v7::createEntry(LinkGraph &G, Symbol &Target) {
+/// Create a new node in the link-graph for the given stub template.
+template <size_t Size>
+static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) {
   constexpr uint64_t Alignment = 4;
-  Block &B = addStub(G, Thumbv7ABS, Alignment);
-  LLVM_DEBUG({
-    const char *StubPtr = B.getContent().data();
-    HalfWords Reg12 = encodeRegMovtT1MovwT3(12);
-    assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
-           checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
-           "Linker generated stubs may only corrupt register r12 (IP)");
-  });
+  ArrayRef<char> Template(reinterpret_cast<const char *>(Code), Size);
+  return G.createContentBlock(S, Template, orc::ExecutorAddr(), Alignment, 0);
+}
+
+static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) {
+  Block &B = allocStub(G, S, Thumbv7ABS);
   B.addEdge(Thumb_MovwAbsNC, 0, Target, 0);
   B.addEdge(Thumb_MovtAbs, 4, Target, 0);
-  Symbol &Stub = G.addAnonymousSymbol(B, 0, B.getSize(), true, false);
-  Stub.setTargetFlags(ThumbSymbol);
-  return Stub;
+
+  [[maybe_unused]] const char *StubPtr = B.getContent().data();
+  [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(12);
+  assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) &&
+         checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) &&
+         "Linker generated stubs may only corrupt register r12 (IP)");
+  return B;
+}
+
+static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) {
+  Block &B = allocStub(G, S, Armv7ABS);
+  B.addEdge(Arm_MovwAbsNC, 0, Target, 0);
+  B.addEdge(Arm_MovtAbs, 4, Target, 0);
+
+  [[maybe_unused]] const char *StubPtr = B.getContent().data();
+  [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(12);
+  assert(checkRegister<Arm_MovwAbsNC>(StubPtr, Reg12) &&
+         checkRegister<Arm_MovtAbs>(StubPtr + 4, Reg12) &&
+         "Linker generated stubs may only corrupt register r12 (IP)");
+  return B;
+}
+
+static bool needsStub(const Edge &E) {
+  Symbol &Target = E.getTarget();
+
+  // Create stubs for external branch targets.
+  if (!Target.isDefined()) {
+    switch (E.getKind()) {
+    case Arm_Call:
+    case Arm_Jump24:
+    case Thumb_Call:
+    case Thumb_Jump24:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  // For local targets, create interworking stubs if we switch Arm/Thumb with an
+  // instruction that cannot switch the instruction set state natively.
+  bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol;
+  switch (E.getKind()) {
+  case Arm_Jump24:
+    return TargetIsThumb; // Branch to Thumb needs interworking stub
+  case Thumb_Jump24:
+    return !TargetIsThumb; // Branch to Arm needs interworking stub
+  default:
+    break;
+  }
+
+  return false;
+}
+
+bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) {
+  if (!needsStub(E))
+    return false;
+
+  // Stub Arm/Thumb follows instruction set state at relocation site.
+  // TODO: We may reduce them at relaxation time and reuse freed slots.
+  bool MakeThumb = (E.getKind() > LastArmRelocation);
+  LLVM_DEBUG(dbgs() << "  Preparing " << (MakeThumb ? "Thumb" : "Arm")
+                    << " stub for " << G.getEdgeKindName(E.getKind())
+                    << " edge at " << B->getFixupAddress(E) << " ("
+                    << B->getAddress() << " + "
+                    << formatv("{0:x}", E.getOffset()) << ")\n");
+
+  Symbol &Target = E.getTarget();
+  assert(Target.hasName() && "Edge cannot point to anonymous target");
+  Symbol *&StubSymbol = getStubSymbolSlot(Target.getName(), MakeThumb);
+
+  if (!StubSymbol) {
+    if (!StubsSection)
+      StubsSection = &G.createSection(getSectionName(),
+                                      orc::MemProt::Read | orc::MemProt::Exec);
+    Block &B = MakeThumb ? createStubThumbv7(G, *StubsSection, Target)
+                         : createStubArmv7(G, *StubsSection, Target);
+    StubSymbol = &G.addAnonymousSymbol(B, 0, B.getSize(), true, false);
+    if (MakeThumb)
+      StubSymbol->setTargetFlags(ThumbSymbol);
+
+    LLVM_DEBUG({
+      dbgs() << "    Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for "
+             << Target.getName() << " in " << StubsSection->getName() << ": "
+             << *StubSymbol << "\n";
+    });
+  }
+
+  assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) &&
+         "Instruction set states of stub and relocation site should be equal");
+  LLVM_DEBUG({
+    dbgs() << "    Using " << (MakeThumb ? "Thumb" : "Arm") << " entry "
+           << *StubSymbol << " in "
+           << StubSymbol->getBlock().getSection().getName() << "\n";
+  });
+
+  E.setTarget(*StubSymbol);
+  return true;
 }
 
 const char *getEdgeKindName(Edge::Kind K) {

llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_arm.s

@@ -0,0 +1,53 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple=armv7-linux-gnueabi -arm-add-build-attributes \
+# RUN:         -filetype=obj -o %t/out.o %s
+# RUN: llvm-jitlink -noexec -slab-address 0x76ff0000 \
+# RUN:              -slab-allocate 10Kb -slab-page-size 4096 \
+# RUN:              -abs ext=0x76bbe880 \
+# RUN:              -check %s %t/out.o
+
+	.text
+	.syntax unified
+
+# Check that calls/jumps to external functions trigger the generation of
+# branch-range extension stubs. These stubs don't follow the default PLT model
+# where the branch-target address is loaded from a GOT entry. Instead, they
+# hard-code it in the immediate field.
+
+# The external function ext will return to the caller directly.
+# jitlink-check: decode_operand(test_arm_jump, 0) = stub_addr(out.o, ext) - (test_arm_jump + 8)
+	.globl	test_arm_jump
+	.type	test_arm_jump,%function
+	.p2align	2
+test_arm_jump:
+	b	ext
+	.size	test_arm_jump, .-test_arm_jump
+
+# The branch-with-link sets the LR register so that the external function ext
+# returns to us. We have to save the register (push) and return to main manually
+# (pop). This adds the +4 offset for the bl instruction we decode:
+# jitlink-check: decode_operand(test_arm_call + 4, 0) = stub_addr(out.o, ext) - (test_arm_call + 8) - 4
+	.globl  test_arm_call
+	.type	test_arm_call,%function
+	.p2align	2
+test_arm_call:
+	push	{lr}
+	bl	ext
+	pop	{pc}
+	.size	test_arm_call, .-test_arm_call
+
+# This test is executable with both, Arm and Thumb `ext` functions. It only has
+# to return with `bx lr`. For example:
+#   > echo "void ext() {}" | clang -target armv7-linux-gnueabihf -o ext-arm.o -c -xc -
+#   > llvm-jitlink ext-arm.o out.o
+#
+	.globl	main
+	.type	main,%function
+	.p2align	2
+main:
+	push	{lr}
+	bl	test_arm_call
+	bl	test_arm_jump
+	movw	r0, #0
+	pop	{pc}
+	.size	main, .-main

llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_multi.s

@@ -0,0 +1,50 @@
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: llvm-mc -triple=armv7-linux-gnueabi -arm-add-build-attributes \
+# RUN:         -filetype=obj -o %t/out.o %s
+# RUN: llvm-jitlink -noexec -slab-address 0x76ff0000 \
+# RUN:              -slab-allocate=10Kb -slab-page-size=4096 \
+# RUN:              -abs ext=0x76bbe880 -check %s %t/out.o
+
+	.text
+	.syntax unified
+
+# Check that a single external symbol can have multiple stubs. We access them
+# with the extra stub-index argument to stub_addr(). Stubs are sorted by
+# ascending size (because the default memory manager lays out blocks by size).
+
+# Thumb relocation site emits thumb stub
+# jitlink-check: decode_operand(test_stub_thumb, 0) = stub_addr(out.o, ext, thumb) - (test_stub_thumb + 4)
+	.globl  test_stub_thumb
+	.type	test_stub_thumb,%function
+	.p2align	1
+	.code	16
+	.thumb_func
+test_stub_thumb:
+	b	ext
+	.size	test_stub_thumb, .-test_stub_thumb
+
+# Arm relocation site emits arm stub
+# jitlink-check: decode_operand(test_stub_arm, 0) = stub_addr(out.o, ext, arm) - (test_stub_arm + 8)
+	.globl  test_stub_arm
+	.type	test_stub_arm,%function
+	.p2align	2
+	.code	32
+test_stub_arm:
+	b	ext
+	.size	test_stub_arm, .-test_stub_arm
+
+# This test is executable with both, Arm and Thumb `ext` functions. It only has
+# to return (directly to main) with `bx lr`. For example:
+#   > echo "void ext() {}" | clang -target armv7-linux-gnueabihf -o ext-arm.o -c -xc -
+#   > llvm-jitlink ext-arm.o out.o
+#
+	.globl	main
+	.type	main,%function
+	.p2align	2
+main:
+	push	{lr}
+	bl	test_stub_arm
+	bl	test_stub_thumb
+	movw	r0, #0
+	pop	{pc}
+	.size	main, .-main

llvm/tools/llvm-jitlink/llvm-jitlink.cpp

@@ -1265,8 +1265,52 @@ Session::findSectionInfo(StringRef FileName, StringRef SectionName) {
   return SecInfoItr->second;
 }
 
+class MemoryMatcher {
+public:
+  MemoryMatcher(ArrayRef<char> Content)
+      : Pos(Content.data()), End(Pos + Content.size()) {}
+
+  template <typename MaskType> bool matchMask(MaskType Mask) {
+    if (Mask == (Mask & *reinterpret_cast<const MaskType *>(Pos))) {
+      Pos += sizeof(MaskType);
+      return true;
+    }
+    return false;
+  }
+
+  template <typename ValueType> bool matchEqual(ValueType Value) {
+    if (Value == *reinterpret_cast<const ValueType *>(Pos)) {
+      Pos += sizeof(ValueType);
+      return true;
+    }
+    return false;
+  }
+
+  bool done() const { return Pos == End; }
+
+private:
+  const char *Pos;
+  const char *End;
+};
+
 static StringRef detectStubKind(const Session::MemoryRegionInfo &Stub) {
-  // Implement acutal stub kind detection
+  constexpr uint32_t Armv7MovWTle = 0xe300c000;
+  constexpr uint32_t Armv7BxR12le = 0xe12fff1c;
+  constexpr uint32_t Thumbv7MovWTle = 0x0c00f240;
+  constexpr uint16_t Thumbv7BxR12le = 0x4760;
+
+  MemoryMatcher M(Stub.getContent());
+  if (M.matchMask(Thumbv7MovWTle)) {
+    if (M.matchMask(Thumbv7MovWTle))
+      if (M.matchEqual(Thumbv7BxR12le))
+        if (M.done())
+          return "thumbv7_abs_le";
+  } else if (M.matchMask(Armv7MovWTle)) {
+    if (M.matchMask(Armv7MovWTle))
+      if (M.matchEqual(Armv7BxR12le))
+        if (M.done())
+          return "armv7_abs_le";
+  }
   return "";
 }