LLDB Debuginfod usage tests (with fixes)

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

@@ -51,7 +51,7 @@ LLDB_BASE_DIR := $(THIS_FILE_DIR)/../../../../../
 #
 # GNUWin32 uname gives "windows32" or "server version windows32" while
 # some versions of MSYS uname return "MSYS_NT*", but most environments
-# standardize on "Windows_NT", so we'll make it consistent here. 
+# standardize on "Windows_NT", so we'll make it consistent here.
 # When running tests from Visual Studio, the environment variable isn't
 # inherited all the way down to the process spawned for make.
 #----------------------------------------------------------------------
@@ -210,6 +210,12 @@ else
 	ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
 		DSYM = $(EXE).debug
 	endif
+
+	ifeq "$(MERGE_DWOS)" "YES"
+		MAKE_DWO := YES
+		DWP_NAME = $(EXE).dwp
+		DYLIB_DWP_NAME = $(DYLIB_NAME).dwp
+	endif
 endif
 
 LIMIT_DEBUG_INFO_FLAGS =
@@ -357,6 +363,7 @@ ifneq "$(OS)" "Darwin"
 
 	OBJCOPY ?= $(call replace_cc_with,objcopy)
 	ARCHIVER ?= $(call replace_cc_with,ar)
+	DWP ?= $(call replace_cc_with,dwp)
 	override AR = $(ARCHIVER)
 endif
 
@@ -565,9 +572,16 @@ else
 endif
 else
 ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
+  ifneq "$(KEEP_FULL_DEBUG_BINARY)" "YES"
 	$(OBJCOPY) --only-keep-debug "$(EXE)" "$(DSYM)"
+  else
+	cp "$(EXE)" "$(DSYM)"
+  endif
 	$(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DSYM)" "$(EXE)" "$(EXE)"
 endif
+ifeq "$(MERGE_DWOS)" "YES"
+	$(DWP) -o "$(DWP_NAME)" $(DWOS)
+endif
 endif
 
 #----------------------------------------------------------------------
@@ -610,9 +624,16 @@ endif
 else
 	$(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -shared -o "$(DYLIB_FILENAME)"
 ifeq "$(SPLIT_DEBUG_SYMBOLS)" "YES"
+	ifneq "$(KEEP_FULL_DEBUG_BINARY)" "YES"
+	cp "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug"
+	else
 	$(OBJCOPY) --only-keep-debug "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME).debug"
+	endif
 	$(OBJCOPY) --strip-debug --add-gnu-debuglink="$(DYLIB_FILENAME).debug" "$(DYLIB_FILENAME)" "$(DYLIB_FILENAME)"
 endif
+ifeq "$(MERGE_DWOS)" "YES"
+	$(DWP) -o $(DYLIB_DWP_FILE) $(DYLIB_DWOS)
+endif
 endif
 
 #----------------------------------------------------------------------

lldb/source/Plugins/SymbolLocator/CMakeLists.txt

@@ -1,5 +1,10 @@
+# Order matters here: the first symbol locator prevents further searching.
+# For DWARF binaries that are both stripped and split, the Default plugin
+# will return the stripped binary when asked for the ObjectFile, which then
+# prevents an unstripped binary from being requested from the Debuginfod
+# provider.
+add_subdirectory(Debuginfod)
 add_subdirectory(Default)
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_subdirectory(DebugSymbols)
 endif()
-add_subdirectory(Debuginfod)

lldb/source/Plugins/SymbolVendor/ELF/SymbolVendorELF.cpp

@@ -44,6 +44,25 @@ llvm::StringRef SymbolVendorELF::GetPluginDescriptionStatic() {
          "executables.";
 }
 
+// If this is needed elsewhere, it can be exported/moved.
+static bool IsDwpSymbolFile(const lldb::ModuleSP &module_sp,
+                            const FileSpec &file_spec) {
+  DataBufferSP dwp_file_data_sp;
+  lldb::offset_t dwp_file_data_offset = 0;
+  // Try to create an ObjectFile from the file_spec.
+  ObjectFileSP dwp_obj_file = ObjectFile::FindPlugin(
+      module_sp, &file_spec, 0, FileSystem::Instance().GetByteSize(file_spec),
+      dwp_file_data_sp, dwp_file_data_offset);
+  if (!ObjectFileELF::classof(dwp_obj_file.get()))
+    return false;
+  // The presence of a debug_cu_index section is the key identifying feature of
+  // a DWP file.
+  if (!dwp_obj_file || !dwp_obj_file->GetSectionList()->FindSectionByType(
+                           eSectionTypeDWARFDebugCuIndex, false))
+    return false;
+  return true;
+}
+
 // CreateInstance
 //
 // Platforms can register a callback to use when creating symbol vendors to
@@ -87,8 +106,15 @@ SymbolVendorELF::CreateInstance(const lldb::ModuleSP &module_sp,
   FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths();
   FileSpec dsym_fspec =
       PluginManager::LocateExecutableSymbolFile(module_spec, search_paths);
-  if (!dsym_fspec)
-    return nullptr;
+  if (!dsym_fspec || IsDwpSymbolFile(module_sp, dsym_fspec)) {
+    // If we have a stripped binary or if we got a DWP file, we should prefer
+    // symbols in the executable acquired through a plugin.
+    ModuleSpec unstripped_spec =
+        PluginManager::LocateExecutableObjectFile(module_spec);
+    if (!unstripped_spec)
+      return nullptr;
+    dsym_fspec = unstripped_spec.GetFileSpec();
+  }
 
   DataBufferSP dsym_file_data_sp;
   lldb::offset_t dsym_file_data_offset = 0;

lldb/test/API/debuginfod/Normal/Makefile

@@ -0,0 +1,32 @@
+C_SOURCES := main.c
+LD_EXTRAS := -Wl,--build-id
+
+# For normal (non DWP) Debuginfod tests, we need:
+
+# * The "full" binary: a.out.debug
+#   Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and
+#   SPLIT_DEBUG_SYMBOLS set to YES
+
+# * The stripped binary (a.out)
+#   Produced by Makefile.rules with SPLIT_DEBUG_SYMBOLS set to YES
+
+# * The 'only-keep-debug' binary (a.out.dbg)
+#   Produced below
+
+# * The .uuid file (for a little easier testing code)
+#   Produced below
+
+# Don't strip the debug info from a.out:
+SPLIT_DEBUG_SYMBOLS := YES
+KEEP_FULL_DEBUG_BINARY := YES
+
+all: a.out a.out.uuid a.out.dbg a.out.debug
+
+# Put the build-id into a file by itself for minimum effort in Python
+a.out.uuid: a.out
+	$(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $<
+
+a.out.dbg: a.out.debug
+	$(OBJCOPY) --only-keep-debug $< $@
+
+include Makefile.rules

lldb/test/API/debuginfod/Normal/TestDebuginfod.py

@@ -0,0 +1,130 @@
+"""
+Test support for the DebugInfoD network symbol acquisition protocol.
+"""
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.lldbtest import *
+
+def getUUID(aoutuuid):
+    """
+    Pull the 20 byte UUID out of the .note.gnu.build-id section that was dumped
+    to a file already, as part of the build.
+    """
+    import struct
+    with open(aoutuuid, "rb") as f:
+        data = f.read(36)
+        if len(data) != 36:
+            return None
+        header = struct.unpack_from("<4I", data)
+        if len(header) != 4:
+            return None
+        # 4 element 'prefix', 20 bytes of uuid, 3 byte long string, 'GNU':
+        if header[0] != 4 or header[1] != 20 or header[2] != 3 or header[3] != 0x554e47:
+            return None
+        return data[16:].hex()
+
+def config_test(local_files, uuid, debuginfo, executable):
+    """
+    Set up a test with local_files[] copied to a particular location
+    so that we control which files are, or are not, found in the file system.
+    Also, create a stand-alone file-system 'hosted' debuginfod server for the
+    given UUID.
+
+    Make the filesystem look like:
+
+    /tmp/<tmpdir>/test/[local_files]
+
+    /tmp/<tmpdir>/cache (for lldb to use as a temp cache)
+
+    /tmp/<tmpdir>/buildid/<uuid>/executable -> <executable>
+    /tmp/<tmpdir>/buildid/<uuid>/debuginfo -> <debuginfo>
+    Returns the /tmp/<tmpdir> path
+    """
+    import os
+    import shutil
+    import tempfile
+
+    tmp_dir = tempfile.mkdtemp()
+    test_dir = os.path.join(tmp_dir, "test")
+    uuid_dir = os.path.join(tmp_dir, "buildid", uuid)
+
+    # Make the 3 directories
+    os.makedirs(os.path.join(tmp_dir, "cache"))
+    os.makedirs(uuid_dir)
+    os.makedirs(test_dir)
+
+    # Copy the files used by the test:
+    for f in local_files:
+        shutil.copy(f, test_dir)
+
+    # Fill in the 'file://...' mocked Debuginfod server
+    if debuginfo:
+        shutil.move(debuginfo, os.path.join(uuiddir, "debuginfo"))
+    if executable:
+        shutil.move(executable, os.path.join(uuiddir, "executable"))
+
+    return tmp_dir
+
+
+# Need to test 5 different scenarios:
+# 1 - A stripped binary with it's corresponding unstripped binary:
+# 2 - A stripped binary with a corresponding --only-keep-debug symbols file
+# 3 - A split binary with it's corresponding DWP file
+# 4 - A stripped, split binary with an unstripped binary and a DWP file
+# 5 - A stripped, split binary with an --only-keep-debug symbols file and a DWP file
+
+class DebugInfodTests(TestBase):
+    # No need to try every flavor of debug inf.
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_stuff(self):
+        """This should test stuff."""
+        self.build()
+        # Pull the UUID out of the binary.
+        uuid_file = self.getBuildArtifact("a.out.uuid")
+        self.aout = self.getBuildArtifact("a.out")
+        self.debugbin = self.getBuildArtifact("a.out.debug")
+        self.dwp = self.getBuildArtifact("a.out.dwp")
+        self.uuid = getUUID(uuid_file)
+        # Setup the fake DebugInfoD server.
+        server_root = config_fake_debuginfod_server(self.uuid, self.dwp, self.debugbin)
+
+        # Configure LLDB properly
+        self.runCmd("settings set symbols.enable-external-lookup true")
+        self.runCmd("settings set plugin.symbol-locator.debuginfod.cache-path %s/cache" % server_root)
+        self.runCmd("settings clear plugin.symbol-locator.debuginfod.server-urls")
+        cmd = "settings insert-before plugin.symbol-locator.debuginfod.server-urls 0 file://%s" % server_root
+        self.runCmd(cmd)
+        print(cmd)
+        # Check to see if the symbol file is properly loaded
+        self.main_source_file = lldb.SBFileSpec("main.c")
+        self.sample_test()
+
+    def sample_test(self):
+        """You might use the test implementation in several ways, say so here."""
+        # This function starts a process, "a.out" by default, sets a source
+        # breakpoint, runs to it, and returns the thread, process & target.
+        # It optionally takes an SBLaunchOption argument if you want to pass
+        # arguments or environment variables.
+        target = target = self.dbg.CreateTarget(self.aout)
+        self.assertTrue(target and target.IsValid(), "Target is valid")
+        bp = target.BreakpointCreateByName("func")
+        self.assertTrue(bp and bp.IsValid(), "Breakpoint is valid")
+        self.assertEqual(bp.GetNumLocations(), 1)
+        print("Loc @ Index 0:")
+        print(bp.GetLocationAtIndex(0))
+        loc = bp.GetLocationAtIndex(0)
+        self.assertTrue(loc and loc.IsValid(), "Location is valid")
+        addr = loc.GetAddress()
+        self.assertTrue(addr and addr.IsValid(), "Loc address is valid")
+        line_entry = addr.GetLineEntry()
+        self.assertTrue(line_entry and line_entry.IsValid(), "Loc line entry is valid")
+        self.assertEqual(line_entry.GetLine(), 18)
+        self.assertEqual(loc.GetLineEntry().GetFileSpec().GetFilename(), self.main_source_file.GetFilename())
+
+    def sample_test_no_launch(self):
+        """Same as above but doesn't launch a process."""
+
+        target = self.createTestTarget()
+        self.expect_expr("global_test_var", result_value="10")

lldb/test/API/debuginfod/Normal/main.c

@@ -0,0 +1,9 @@
+// This is a dump little pair of test files
+
+int func(int argc, const char *argv[]) {
+  return (argc + 1) * (argv[argc][0] + 2);
+}
+
+int main(int argc, const char *argv[]) {
+  return func(0, argv);
+}

lldb/test/API/debuginfod/SplitDWARF/Makefile

@@ -0,0 +1,36 @@
+C_SOURCES := main.c
+LD_EXTRAS := -Wl,--build-id
+
+# For split-dwarf Debuginfod tests, we need:
+
+# * A .DWP file (a.out.dwp)
+#   Produced by Makefile.rules with MAKE_DWO and MERGE_DWOS both set to YES
+
+# * The "full" binary: it's missing things that live in .dwo's (a.out.debug)
+#   Produced by Makefile.rules with KEEP_FULL_DEBUG_BINARY set to YES and
+#   SPLIT_DEBUG_SYMBOLS set to YES
+
+# * The stripped binary (a.out)
+#   Produced by Makefile.rules
+
+# * The 'only-keep-debug' binary (a.out.dbg)
+#   Produced below
+
+# * The .uuid file (for a little easier testing code)
+#   Produced here in the rule below
+
+MAKE_DWO := YES
+MERGE_DWOS := YES
+SPLIT_DEBUG_SYMBOLS := YES
+KEEP_FULL_DEBUG_BINARY := YES
+
+all: a.out.uuid a.out a.out.debug a.out.dbg
+
+a.out.dbg: a.out.debug
+	$(OBJCOPY) --only-keep-debug $< $@
+
+# Put the build-id into a file by itself for minimum effort in Python
+a.out.uuid: a.out
+	$(OBJCOPY) --dump-section=.note.gnu.build-id=$@ $<
+
+include Makefile.rules