Improve code generation for thread_local variables:

Summary:
 * Don't bother using a thread wrapper when the variable is known to
   have constant initialization.
 * Emit the thread wrapper as discardable-if-unused in TUs that don't
   contain a definition of the thread_local variable.
 * Don't emit the thread wrapper at all if the thread_local variable
   is unused and discardable; it will be emitted by all TUs that need
   it.

Reviewers: rjmccall, jdoerfert

Subscribers: cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D67429

llvm-svn: 371767

Author: zygoloid

clang/include/clang/Basic/Linkage.h

@@ -82,6 +82,12 @@ inline bool isDiscardableGVALinkage(GVALinkage L) {
   return L <= GVA_DiscardableODR;
 }
 
+/// Do we know that this will be the only definition of this symbol (excluding
+/// inlining-only definitions)?
+inline bool isUniqueGVALinkage(GVALinkage L) {
+  return L == GVA_Internal || L == GVA_StrongExternal;
+}
+
 inline bool isExternallyVisible(Linkage L) {
   return L >= VisibleNoLinkage;
 }

clang/lib/CodeGen/CGCXXABI.h

@@ -577,7 +577,7 @@ class CGCXXABI {
 
   // Determine if references to thread_local global variables can be made
   // directly or require access through a thread wrapper function.
-  virtual bool usesThreadWrapperFunction() const = 0;
+  virtual bool usesThreadWrapperFunction(const VarDecl *VD) const = 0;
 
   /// Emit a reference to a non-local thread_local variable (including
   /// triggering the initialization of all thread_local variables in its

clang/lib/CodeGen/CGExpr.cpp

@@ -2361,7 +2361,7 @@ static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
 
   // If it's thread_local, emit a call to its wrapper function instead.
   if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
-      CGF.CGM.getCXXABI().usesThreadWrapperFunction())
+      CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
     return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
   // Check if the variable is marked as declare target with link clause in
   // device codegen.

clang/lib/CodeGen/ItaniumCXXABI.cpp

@@ -43,6 +43,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
   /// VTables - All the vtables which have been defined.
   llvm::DenseMap<const CXXRecordDecl *, llvm::GlobalVariable *> VTables;
 
+  /// All the thread wrapper functions that have been used.
+  llvm::SmallVector<std::pair<const VarDecl *, llvm::Function *>, 8>
+      ThreadWrappers;
+
 protected:
   bool UseARMMethodPtrABI;
   bool UseARMGuardVarABI;
@@ -322,7 +326,42 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
       ArrayRef<llvm::Function *> CXXThreadLocalInits,
       ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
 
-  bool usesThreadWrapperFunction() const override { return true; }
+  /// Determine whether we will definitely emit this variable with a constant
+  /// initializer, either because the language semantics demand it or because
+  /// we know that the initializer is a constant.
+  bool isEmittedWithConstantInitializer(const VarDecl *VD) const {
+    VD = VD->getMostRecentDecl();
+    if (VD->hasAttr<ConstInitAttr>())
+      return true;
+
+    // All later checks examine the initializer specified on the variable. If
+    // the variable is weak, such examination would not be correct.
+    if (VD->isWeak() || VD->hasAttr<SelectAnyAttr>())
+      return false;
+
+    const VarDecl *InitDecl = VD->getInitializingDeclaration();
+    if (!InitDecl)
+      return false;
+
+    // If there's no initializer to run, this is constant initialization.
+    if (!InitDecl->hasInit())
+      return true;
+
+    // If we have the only definition, we don't need a thread wrapper if we
+    // will emit the value as a constant.
+    if (isUniqueGVALinkage(getContext().GetGVALinkageForVariable(VD)))
+      return !VD->getType().isDestructedType() && InitDecl->evaluateValue();
+
+    // Otherwise, we need a thread wrapper unless we know that every
+    // translation unit will emit the value as a constant. We rely on
+    // ICE-ness not varying between translation units, which isn't actually
+    // guaranteed by the standard but is necessary for sanity.
+    return InitDecl->isInitKnownICE() && InitDecl->isInitICE();
+  }
+
+  bool usesThreadWrapperFunction(const VarDecl *VD) const override {
+    return !isEmittedWithConstantInitializer(VD);
+  }
   LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
                                       QualType LValType) override;
 
@@ -2456,9 +2495,6 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
 
   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Wrapper);
 
-  if (VD->hasDefinition())
-    CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Wrapper);
-
   // Always resolve references to the wrapper at link time.
   if (!Wrapper->hasLocalLinkage())
     if (!isThreadWrapperReplaceable(VD, CGM) ||
@@ -2471,6 +2507,8 @@ ItaniumCXXABI::getOrCreateThreadLocalWrapper(const VarDecl *VD,
     Wrapper->setCallingConv(llvm::CallingConv::CXX_FAST_TLS);
     Wrapper->addFnAttr(llvm::Attribute::NoUnwind);
   }
+
+  ThreadWrappers.push_back({VD, Wrapper});
   return Wrapper;
 }
 
@@ -2519,20 +2557,40 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
     }
   }
 
-  // Emit thread wrappers.
+  // Create declarations for thread wrappers for all thread-local variables
+  // with non-discardable definitions in this translation unit.
   for (const VarDecl *VD : CXXThreadLocals) {
+    if (VD->hasDefinition() &&
+        !isDiscardableGVALinkage(getContext().GetGVALinkageForVariable(VD))) {
+      llvm::GlobalValue *GV = CGM.GetGlobalValue(CGM.getMangledName(VD));
+      getOrCreateThreadLocalWrapper(VD, GV);
+    }
+  }
+
+  // Emit all referenced thread wrappers.
+  for (auto VDAndWrapper : ThreadWrappers) {
+    const VarDecl *VD = VDAndWrapper.first;
     llvm::GlobalVariable *Var =
         cast<llvm::GlobalVariable>(CGM.GetGlobalValue(CGM.getMangledName(VD)));
-    llvm::Function *Wrapper = getOrCreateThreadLocalWrapper(VD, Var);
+    llvm::Function *Wrapper = VDAndWrapper.second;
 
     // Some targets require that all access to thread local variables go through
     // the thread wrapper.  This means that we cannot attempt to create a thread
     // wrapper or a thread helper.
-    if (isThreadWrapperReplaceable(VD, CGM) && !VD->hasDefinition()) {
-      Wrapper->setLinkage(llvm::Function::ExternalLinkage);
-      continue;
+    if (!VD->hasDefinition()) {
+      if (isThreadWrapperReplaceable(VD, CGM)) {
+        Wrapper->setLinkage(llvm::Function::ExternalLinkage);
+        continue;
+      }
+
+      // If this isn't a TU in which this variable is defined, the thread
+      // wrapper is discardable.
+      if (Wrapper->getLinkage() == llvm::Function::WeakODRLinkage)
+        Wrapper->setLinkage(llvm::Function::LinkOnceODRLinkage);
     }
 
+    CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Wrapper);
+
     // Mangle the name for the thread_local initialization function.
     SmallString<256> InitFnName;
     {
@@ -2547,7 +2605,10 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
     // produce a declaration of the initialization function.
     llvm::GlobalValue *Init = nullptr;
     bool InitIsInitFunc = false;
-    if (VD->hasDefinition()) {
+    bool HasConstantInitialization = false;
+    if (isEmittedWithConstantInitializer(VD)) {
+      HasConstantInitialization = true;
+    } else if (VD->hasDefinition()) {
       InitIsInitFunc = true;
       llvm::Function *InitFuncToUse = InitFunc;
       if (isTemplateInstantiation(VD->getTemplateSpecializationKind()))
@@ -2576,7 +2637,9 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
     llvm::LLVMContext &Context = CGM.getModule().getContext();
     llvm::BasicBlock *Entry = llvm::BasicBlock::Create(Context, "", Wrapper);
     CGBuilderTy Builder(CGM, Entry);
-    if (InitIsInitFunc) {
+    if (HasConstantInitialization) {
+      // No dynamic initialization to invoke.
+    } else if (InitIsInitFunc) {
       if (Init) {
         llvm::CallInst *CallVal = Builder.CreateCall(InitFnTy, Init);
         if (isThreadWrapperReplaceable(VD, CGM)) {

clang/lib/CodeGen/MicrosoftCXXABI.cpp

@@ -386,7 +386,9 @@ class MicrosoftCXXABI : public CGCXXABI {
       ArrayRef<llvm::Function *> CXXThreadLocalInits,
       ArrayRef<const VarDecl *> CXXThreadLocalInitVars) override;
 
-  bool usesThreadWrapperFunction() const override { return false; }
+  bool usesThreadWrapperFunction(const VarDecl *VD) const override {
+    return false;
+  }
   LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
                                       QualType LValType) override;
 

clang/test/CodeGenCXX/cxx11-thread-local.cpp

@@ -91,15 +91,16 @@ void *e2 = V<char>::m + W<char>::m + &X<char>::m;
 
 // LINUX-DAG: @_ZTH1a = alias void (), void ()* @__tls_init
 // DARWIN-DAG: @_ZTH1a = internal alias void (), void ()* @__tls_init
-// CHECK-DAG: @_ZTHL1d = internal alias void (), void ()* @__tls_init
 // LINUX-DAG: @_ZTHN1U1mE = alias void (), void ()* @__tls_init
 // DARWIN-DAG: @_ZTHN1U1mE = internal alias void (), void ()* @__tls_init
 // CHECK-DAG: @_ZTHN1VIiE1mE = linkonce_odr alias void (), void ()* @[[V_M_INIT:[^, ]*]]
-// CHECK-NOT: @_ZTHN1WIiE1mE =
 // CHECK-DAG: @_ZTHN1XIiE1mE = linkonce_odr alias void (), void ()* @[[X_M_INIT:[^, ]*]]
 // CHECK-DAG: @_ZTHN1VIfE1mE = weak_odr alias void (), void ()* @[[VF_M_INIT:[^, ]*]]
-// CHECK-NOT: @_ZTHN1WIfE1mE =
 // CHECK-DAG: @_ZTHN1XIfE1mE = weak_odr alias void (), void ()* @[[XF_M_INIT:[^, ]*]]
+// FIXME: We really want a CHECK-DAG-NOT for these.
+// CHECK-NOT: @_ZTHN1WIiE1mE =
+// CHECK-NOT: @_ZTHN1WIfE1mE =
+// CHECK-NOT: @_ZTHL1d =
 
 
 // Individual variable initialization functions:
@@ -130,7 +131,7 @@ int f() {
 // CHECK-NEXT: load i32, i32* %{{.*}}, align 4
 // CHECK-NEXT: store i32 %{{.*}}, i32* @c, align 4
 
-// LINUX-LABEL: define weak_odr hidden i32* @_ZTW1b()
+// LINUX-LABEL: define linkonce_odr hidden i32* @_ZTW1b()
 // LINUX: br i1 icmp ne (void ()* @_ZTH1b, void ()* null),
 // not null:
 // LINUX: call void @_ZTH1b()
@@ -203,21 +204,21 @@ int f() {
 // DARWIN: declare i32 @_tlv_atexit(void (i8*)*, i8*, i8*)
 
 // DARWIN: declare cxx_fast_tlscc i32* @_ZTWN1VIcE1mE()
-// LINUX: define weak_odr hidden i32* @_ZTWN1VIcE1mE()
+// LINUX: define linkonce_odr hidden i32* @_ZTWN1VIcE1mE()
 // LINUX-NOT: comdat
 // LINUX: br i1 icmp ne (void ()* @_ZTHN1VIcE1mE,
 // LINUX: call void @_ZTHN1VIcE1mE()
 // LINUX: ret i32* @_ZN1VIcE1mE
 
 // DARWIN: declare cxx_fast_tlscc i32* @_ZTWN1WIcE1mE()
-// LINUX: define weak_odr hidden i32* @_ZTWN1WIcE1mE()
+// LINUX: define linkonce_odr hidden i32* @_ZTWN1WIcE1mE()
 // LINUX-NOT: comdat
 // LINUX: br i1 icmp ne (void ()* @_ZTHN1WIcE1mE,
 // LINUX: call void @_ZTHN1WIcE1mE()
 // LINUX: ret i32* @_ZN1WIcE1mE
 
 // DARWIN: declare cxx_fast_tlscc {{.*}}* @_ZTWN1XIcE1mE()
-// LINUX: define weak_odr hidden {{.*}}* @_ZTWN1XIcE1mE()
+// LINUX: define linkonce_odr hidden {{.*}}* @_ZTWN1XIcE1mE()
 // LINUX-NOT: comdat
 // LINUX: br i1 icmp ne (void ()* @_ZTHN1XIcE1mE,
 // LINUX: call void @_ZTHN1XIcE1mE()
@@ -269,7 +270,7 @@ int PR19254::f() {
 }
 
 namespace {
-thread_local int anon_i{1};
+thread_local int anon_i{f()};
 }
 void set_anon_i() {
   anon_i = 2;
@@ -332,19 +333,17 @@ void set_anon_i() {
 // CHECK: }
 
 
-// LINUX: declare extern_weak void @_ZTH1b() [[ATTR:#[0-9]+]]
-
-
-// LINUX-LABEL: define internal i32* @_ZTWL1d()
-// DARWIN-LABEL: define internal cxx_fast_tlscc i32* @_ZTWL1d()
-// LINUX: call void @_ZTHL1d()
-// DARWIN: call cxx_fast_tlscc void @_ZTHL1d()
-// CHECK: ret i32* @_ZL1d
+// Should not emit a thread wrapper for internal-linkage unused variable 'd'.
+// We separately check that 'd' does in fact get initialized with the other
+// thread-local variables in this TU.
+// CHECK-NOT: define {{.*}} @_ZTWL1d()
 
 // LINUX-LABEL: define weak_odr hidden i32* @_ZTWN1U1mE()
 // DARWIN-LABEL: define cxx_fast_tlscc i32* @_ZTWN1U1mE()
 // LINUX: call void @_ZTHN1U1mE()
 // DARWIN: call cxx_fast_tlscc void @_ZTHN1U1mE()
 // CHECK: ret i32* @_ZN1U1mE
 
+// LINUX: declare extern_weak void @_ZTH1b() [[ATTR:#[0-9]+]]
+
 // LINUX: attributes [[ATTR]] = { {{.+}} }

clang/test/CodeGenCXX/cxx2a-thread-local-constinit.cpp

@@ -0,0 +1,47 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -std=c++2a %s -emit-llvm -o - | FileCheck %s
+
+// CHECK-DAG: @a = external thread_local global i32
+extern thread_local int a;
+
+// CHECK-DAG: @b = external thread_local global i32
+extern thread_local constinit int b;
+
+// CHECK-LABEL: define i32 @_Z1fv()
+// CHECK: call i32* @_ZTW1a()
+// CHECK: }
+int f() { return a; }
+
+// CHECK-LABEL: define linkonce_odr {{.*}} @_ZTW1a()
+// CHECK: br i1
+// CHECK: call void @_ZTH1a()
+// CHECK: }
+
+// CHECK-LABEL: define i32 @_Z1gv()
+// CHECK-NOT: call
+// CHECK: load i32, i32* @b
+// CHECK-NOT: call
+// CHECK: }
+int g() { return b; }
+
+// CHECK-NOT: define {{.*}} @_ZTW1b()
+
+extern thread_local int c;
+
+// CHECK-LABEL: define i32 @_Z1hv()
+// CHECK: call i32* @_ZTW1c()
+// CHECK: load i32, i32* %
+// CHECK: }
+int h() { return c; }
+
+thread_local int c = 0;
+
+int d_init();
+thread_local int d = d_init();
+
+// Note: use of 'c' does not trigger initialization of 'd', because 'c' has a
+// constant initializer.
+// CHECK-LABEL: define weak_odr {{.*}} @_ZTW1c()
+// CHECK-NOT: br i1
+// CHECK-NOT: call
+// CHECK: ret i32* @c
+// CHECK: }

clang/test/CodeGenCXX/tls-init-funcs.cpp

@@ -6,9 +6,9 @@
 // CHECK: @_tlv_atexit({{.*}}@_ZN1AD1Ev
 // CHECK: call cxx_fast_tlscc i32* @_ZTW3ext()
 // CHECK: declare cxx_fast_tlscc i32* @_ZTW3ext()
-// CHECK: define weak_odr hidden cxx_fast_tlscc i32* @_ZTW2vtIiE()
-// CHECK: define weak_odr hidden cxx_fast_tlscc i32* @_ZTW2vtIvE()
-// CHECK: define {{.*}} @_ZTW1a
+// CHECK-DAG: define weak_odr hidden cxx_fast_tlscc i32* @_ZTW2vtIiE()
+// CHECK-DAG: define weak_odr hidden cxx_fast_tlscc i32* @_ZTW2vtIvE()
+// CHECK-DAG: define {{.*}} @_ZTW1a
 
 struct A {
   ~A();

clang/test/CodeGen/windows-on-arm-itanium-thread-local.c renamed to clang/test/CodeGenCXX/windows-on-arm-itanium-thread-local.cpp

@@ -1,11 +1,12 @@
-// RUN: %clang_cc1 -triple thumbv7--windows-itanium -fdeclspec -fms-compatibility -fms-compatibility-version=19.0 -S -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv7--windows-itanium -fdeclspec -fms-compatibility -fms-compatibility-version=19.0 -emit-llvm -o - %s | FileCheck %s
 
-__declspec(thread) static void *c;
+void *g();
+thread_local static void *c = g();
 void f(void *p) {
   c = p;
 }
 
-// CHECK-LABEL: @f(i8* %p)
+// CHECK-LABEL: @_Z1fPv(i8* %p)
 // CHECK-NOT: call i8** @_ZTWL1c()
 // CHECK: call arm_aapcs_vfpcc i8** @_ZTWL1c()