[dsymutil] Teach dsymutil about CAS and loading clang modules from CAS

Teach dsymutil how to use CAS and how to load clang modules from CAS
when building dSYM when gmodule is used.

Author: cachemeifyoucan

clang/test/CMakeLists.txt

@@ -155,6 +155,7 @@ if( NOT CLANG_BUILT_STANDALONE )
     llvm-config
     FileCheck count not
     CASPluginTest
+    dsymutil
     llc
     llvm-ar
     llvm-as

clang/test/ClangScanDeps/gmodules.c

@@ -35,6 +35,26 @@
 // RUN: %clang @%t/Right.rsp
 // RUN: %clang @%t/tu.rsp
 
+// RUN: llvm-dwarfdump --debug-info %t/tu.o | FileCheck %s --check-prefix=OBJECT-DWARF
+// OBJECT-DWARF: DW_TAG_compile_unit
+// OBJECT-DWARF: DW_AT_name        ("Left")
+// OBJECT-DWARF: DW_AT_dwo_name    ("llvmcas://
+
+/// Check debug info is correct.
+// RUN: %clang %t/tu.o -o %t/a.out
+// RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty
+// RUN: dsymutil %t/a.out -o %t/a2.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty
+// WARN-NOT: warning:
+
+// RUN: dsymutil -cas %t/cas-empty %t/a.out -o %t/a3.dSYM 2>&1 | FileCheck %s --check-prefix=WARN-CAS --check-prefix=WARN-FILE
+// WARN-CAS: warning: failed to load CAS object
+// RUN: echo "bad" > %t/.cas-config
+// RUN: dsymutil %t/a.out -o %t/a4.dSYM 2>&1 | FileCheck %s --check-prefix=WARN-FILE
+// WARN-FILE: warning:
+// WARN-FILE-SAME: No such file or directory
+
+// RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF
+
 /// Check module in a different directory and compare output.
 // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \
 // RUN:   -cas-path %t/cas -module-files-dir %t/outputs-2 \
@@ -59,6 +79,39 @@
 // RUN: diff %t/prefix.h.pch %t/prefix_2.pch
 // RUN: diff %t/tu.o %t/tu_2.o
 
+// Check all the types are available
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Top")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Top")
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Left")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Left")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Right")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Right")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Prefix")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+
 
 // CHECK:      {
 // CHECK-NEXT:  "modules": [

llvm/docs/CommandGuide/dsymutil.rst

@@ -40,6 +40,19 @@ OPTIONS
  'profile'. Setting the DYLD_IMAGE_SUFFIX environment variable will
  cause dyld to load the specified variant at runtime.
 
+.. option:: --cas <path to CAS>
+
+ Specify the path to a content addressable storage that dsymutil may used to
+ resolve CASID file paths in the debug info.
+
+.. option:: --cas-plugin-path <path to dylib>
+
+ Specify the path to CAS plugin dylib if used.
+
+.. option:: --cas-plugin-option <cas option>
+
+ Specify the options to pass to CAS plugin dylib if needed.
+
 .. option:: --dump-debug-map
 
  Dump the *executable*'s debug-map (the list of the object files containing the

llvm/include/llvm/DWARFLinker/Classic/DWARFLinker.h

@@ -239,7 +239,8 @@ class LLVM_ABI DWARFLinker : public DWARFLinkerBase {
   /// \pre NoODR, Update options should be set before call to addObjectFile.
   void addObjectFile(
       DWARFFile &File, ObjFileLoaderTy Loader = nullptr,
-      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {}) override;
+      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {},
+      CASLoaderTy CASLoader = nullptr) override;
 
   /// Link debug info for added objFiles. Object files are linked all together.
   Error link() override;
@@ -487,15 +488,15 @@ class LLVM_ABI DWARFLinker : public DWARFLinkerBase {
   bool registerModuleReference(const DWARFDie &CUDie, LinkContext &Context,
                                ObjFileLoaderTy Loader,
                                CompileUnitHandlerTy OnCUDieLoaded,
-                               unsigned Indent = 0);
+                               CASLoaderTy CASLoader, unsigned Indent = 0);
 
   /// Recursively add the debug info in this clang module .pcm
   /// file (and all the modules imported by it in a bottom-up fashion)
   /// to ModuleUnits.
   Error loadClangModule(ObjFileLoaderTy Loader, const DWARFDie &CUDie,
                         const std::string &PCMFile, LinkContext &Context,
                         CompileUnitHandlerTy OnCUDieLoaded,
-                        unsigned Indent = 0);
+                        CASLoaderTy CASLoader, unsigned Indent = 0);
 
   /// Clone specified Clang module unit \p Unit.
   Error cloneModuleUnit(LinkContext &Context, RefModuleUnit &Unit,

llvm/include/llvm/DWARFLinker/DWARFLinkerBase.h

@@ -84,6 +84,8 @@ class DWARFLinkerBase {
   using ObjectPrefixMapTy = std::map<std::string, std::string>;
   using CompileUnitHandlerTy = function_ref<void(const DWARFUnit &Unit)>;
   using SwiftInterfacesMapTy = std::map<std::string, std::string>;
+  using CASLoaderTy =
+      std::function<Expected<DWARFFile *>(StringRef CASID, StringRef Filename)>;
   /// Type of output file.
   enum class OutputFileType : uint8_t {
     Object,
@@ -99,12 +101,15 @@ class DWARFLinkerBase {
   /// \p OnCUDieLoaded for each compile unit die. If \p File has reference to
   /// a Clang module and UpdateIndexTablesOnly == false then the module is be
   /// pre-loaded by \p Loader.
+  /// \p CASLoader for loading file from CAS when compilation caching is
+  /// enabled.
   ///
   /// \pre a call to setNoODR(true) and/or setUpdateIndexTablesOnly(bool Update)
   ///      must be made when required.
   virtual void addObjectFile(
       DWARFFile &File, ObjFileLoaderTy Loader = nullptr,
-      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {}) = 0;
+      CompileUnitHandlerTy OnCUDieLoaded = [](const DWARFUnit &) {},
+      CASLoaderTy CASLoader = nullptr) = 0;
   /// Link the debug info for all object files added through calls to
   /// addObjectFile.
   virtual Error link() = 0;

llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp

@@ -2611,6 +2611,7 @@ bool DWARFLinker::registerModuleReference(const DWARFDie &CUDie,
                                           LinkContext &Context,
                                           ObjFileLoaderTy Loader,
                                           CompileUnitHandlerTy OnCUDieLoaded,
+                                          CASLoaderTy CASLoader,
                                           unsigned Indent) {
   std::string PCMFile = getPCMFile(CUDie, Options.ObjectPrefixMap);
   std::pair<bool, bool> IsClangModuleRef =
@@ -2630,47 +2631,62 @@ bool DWARFLinker::registerModuleReference(const DWARFDie &CUDie,
   ClangModules.insert({PCMFile, getDwoId(CUDie)});
 
   if (Error E = loadClangModule(Loader, CUDie, PCMFile, Context, OnCUDieLoaded,
-                                Indent + 2)) {
+                                CASLoader, Indent + 2)) {
     consumeError(std::move(E));
     return false;
   }
   return true;
 }
 
-Error DWARFLinker::loadClangModule(
-    ObjFileLoaderTy Loader, const DWARFDie &CUDie, const std::string &PCMFile,
-    LinkContext &Context, CompileUnitHandlerTy OnCUDieLoaded, unsigned Indent) {
+Error DWARFLinker::loadClangModule(ObjFileLoaderTy Loader,
+                                   const DWARFDie &CUDie,
+                                   const std::string &PCMFile,
+                                   LinkContext &Context,
+                                   CompileUnitHandlerTy OnCUDieLoaded,
+                                   CASLoaderTy CASLoader, unsigned Indent) {
 
   uint64_t DwoId = getDwoId(CUDie);
   std::string ModuleName = dwarf::toString(CUDie.find(dwarf::DW_AT_name), "");
 
-  /// Using a SmallString<0> because loadClangModule() is recursive.
-  SmallString<0> Path(Options.PrependPath);
-  if (sys::path::is_relative(PCMFile))
-    resolveRelativeObjectPath(Path, CUDie);
-  sys::path::append(Path, PCMFile);
-  // Don't use the cached binary holder because we have no thread-safety
-  // guarantee and the lifetime is limited.
-
-  if (Loader == nullptr) {
-    reportError("Could not load clang module: loader is not specified.\n",
-                Context.File);
-    return Error::success();
-  }
+  DWARFFile *PCM = nullptr;
+  if (CASLoader) {
+    auto LoadPCM = CASLoader(PCMFile, Context.File.FileName);
+    if (!LoadPCM)
+      return LoadPCM.takeError();
+    PCM = *LoadPCM;
+  }
+
+  if (!PCM) {
+    /// Using a SmallString<0> because loadClangModule() is recursive.
+    SmallString<0> Path(Options.PrependPath);
+    if (sys::path::is_relative(PCMFile))
+      resolveRelativeObjectPath(Path, CUDie);
+    sys::path::append(Path, PCMFile);
+    // Don't use the cached binary holder because we have no thread-safety
+    // guarantee and the lifetime is limited.
+
+    if (Loader == nullptr) {
+      reportError("Could not load clang module: loader is not specified.\n",
+                  Context.File);
+      return Error::success();
+    }
 
-  auto ErrOrObj = Loader(Context.File.FileName, Path);
-  if (!ErrOrObj)
-    return Error::success();
+    auto ErrOrObj = Loader(Context.File.FileName, Path);
+    if (!ErrOrObj)
+      return Error::success();
+
+    PCM = &*ErrOrObj;
+  }
 
   std::unique_ptr<CompileUnit> Unit;
-  for (const auto &CU : ErrOrObj->Dwarf->compile_units()) {
+  for (const auto &CU : PCM->Dwarf->compile_units()) {
     OnCUDieLoaded(*CU);
     // Recursively get all modules imported by this one.
     auto ChildCUDie = CU->getUnitDIE();
     if (!ChildCUDie)
       continue;
     if (!registerModuleReference(ChildCUDie, Context, Loader, OnCUDieLoaded,
-                                 Indent)) {
+                                 CASLoader, Indent)) {
       if (Unit) {
         std::string Err =
             (PCMFile +
@@ -2700,7 +2716,7 @@ Error DWARFLinker::loadClangModule(
   }
 
   if (Unit)
-    Context.ModuleUnits.emplace_back(RefModuleUnit{*ErrOrObj, std::move(Unit)});
+    Context.ModuleUnits.emplace_back(RefModuleUnit{*PCM, std::move(Unit)});
 
   return Error::success();
 }
@@ -2807,7 +2823,8 @@ void DWARFLinker::copyInvariantDebugSection(DWARFContext &Dwarf) {
 }
 
 void DWARFLinker::addObjectFile(DWARFFile &File, ObjFileLoaderTy Loader,
-                                CompileUnitHandlerTy OnCUDieLoaded) {
+                                CompileUnitHandlerTy OnCUDieLoaded,
+                                CASLoaderTy CASLoader) {
   ObjectContexts.emplace_back(LinkContext(File));
 
   if (ObjectContexts.back().File.Dwarf) {
@@ -2822,7 +2839,7 @@ void DWARFLinker::addObjectFile(DWARFFile &File, ObjFileLoaderTy Loader,
 
       if (!LLVM_UNLIKELY(Options.Update))
         registerModuleReference(CUDie, ObjectContexts.back(), Loader,
-                                OnCUDieLoaded);
+                                OnCUDieLoaded, CASLoader);
     }
   }
 }