[dsymutil] Teach dsymutil about CAS and loading clang modules from CAS

Teach dsymutil how to use CAS and how to load clang modules from CAS
when building dSYM when gmodule is used.

Author: cachemeifyoucan

clang/test/CMakeLists.txt

@@ -155,6 +155,7 @@ if( NOT CLANG_BUILT_STANDALONE )
     llvm-config
     FileCheck count not
     CASPluginTest
+    dsymutil
     llc
     llvm-ar
     llvm-as

clang/test/ClangScanDeps/gmodules.c

@@ -35,6 +35,14 @@
 // RUN: %clang @%t/Right.rsp
 // RUN: %clang @%t/tu.rsp
 
+/// Check debug info is correct.
+// RUN: %clang %t/tu.o -o %t/a.out
+// RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty
+// RUN: dsymutil %t/a.out -o %t/a2.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty
+// WARN-NOT: warning:
+
+// RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF
+
 /// Check module in a different directory and compare output.
 // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \
 // RUN:   -cas-path %t/cas -module-files-dir %t/outputs-2 \
@@ -59,6 +67,39 @@
 // RUN: diff %t/prefix.h.pch %t/prefix_2.pch
 // RUN: diff %t/tu.o %t/tu_2.o
 
+// Check all the types are available
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Top")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Top")
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Left")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Left")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF-NEXT: DW_AT_name      ("Right")
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Right")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+// DWARF: DW_TAG_compile_unit
+// DWARF: DW_TAG_module
+// DWARF: DW_TAG_structure_type
+// DWARF-NEXT: DW_AT_name    ("Prefix")
+// DWARF: DW_TAG_member
+// DWARF-NEXT: DW_AT_name  ("top")
+// DWARF-NEXT: DW_AT_type
+// DWARF-SAME: "Top::Top"
+
 
 // CHECK:      {
 // CHECK-NEXT:  "modules": [

llvm/tools/dsymutil/BinaryHolder.cpp

@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "BinaryHolder.h"
+#include "llvm/CAS/CASConfiguration.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
@@ -42,8 +43,9 @@ getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem,
 }
 
 BinaryHolder::BinaryHolder(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
-                           BinaryHolder::Options Opts)
-    : VFS(VFS), Opts(Opts) {}
+                           BinaryHolder::Options Opts,
+                           std::shared_ptr<cas::ObjectStore> CAS)
+    : VFS(VFS), CAS(std::move(CAS)), Opts(Opts) {}
 
 Error BinaryHolder::ArchiveEntry::load(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
                                        StringRef Filename,
@@ -144,6 +146,37 @@ Error BinaryHolder::ObjectEntry::load(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
   return Error::success();
 }
 
+bool BinaryHolder::ObjectEntry::load(cas::ObjectStore &CAS,
+                                     StringRef PossibleID, Options Opts) {
+  auto ID = CAS.parseID(PossibleID);
+  if (!ID) {
+    consumeError(ID.takeError());
+    return false;
+  }
+
+  if (Opts.Verbose)
+    WithColor::note() << "try load cas object " << PossibleID << "\n";
+
+  auto Ref = CAS.getProxy(*ID);
+  if (!Ref) {
+    WithColor::warning() << "failed to load CAS object " << PossibleID << "\n";
+    return false;
+  }
+
+  // Only module files should be loaded here. Always not fat.
+  auto ObjectBuf = Ref->getMemoryBuffer();
+  auto ErrOrObjectFile =
+      object::ObjectFile::createObjectFile(ObjectBuf->getMemBufferRef());
+  if (!ErrOrObjectFile) {
+    WithColor::warning() << "malformed object loaded via " << PossibleID
+                         << "\n";
+    return false;
+  }
+
+  Objects.push_back(std::move(*ErrOrObjectFile));
+  return true;
+}
+
 std::vector<const object::ObjectFile *>
 BinaryHolder::ObjectEntry::getObjects() const {
   std::vector<const object::ObjectFile *> Result;
@@ -231,6 +264,22 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename,
   return *(MemberCache[Key] = std::move(OE));
 }
 
+static std::shared_ptr<cas::ObjectStore> searchAndCreateCAS(StringRef Path) {
+  auto Config = cas::CASConfiguration::createFromSearchConfigFile(Path);
+  if (!Config)
+    return nullptr;
+
+  auto DB = Config->second.createDatabases();
+  if (!DB) {
+    consumeError(DB.takeError());
+    return nullptr;
+  }
+
+  WithColor::note() << "create CAS using configuration: " << Config->first
+                    << "'\n";
+  return DB->first;
+}
+
 Expected<const BinaryHolder::ObjectEntry &>
 BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) {
   if (Opts.Verbose)
@@ -265,9 +314,14 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) {
   ObjectRefCounter[Filename]++;
   if (!ObjectCache.count(Filename)) {
     auto OE = std::make_unique<ObjectEntry>();
-    auto Err = OE->load(VFS, Filename, Timestamp, Opts);
-    if (Err)
-      return std::move(Err);
+    if (!CAS)
+      CAS = searchAndCreateCAS(Filename);
+
+    if (!(CAS && OE->load(*CAS, Filename, Opts))) {
+      auto Err = OE->load(VFS, Filename, Timestamp, Opts);
+      if (Err)
+        return std::move(Err);
+    }
     ObjectCache[Filename] = std::move(OE);
   }
 

llvm/tools/dsymutil/BinaryHolder.h

@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/MachOUniversal.h"
@@ -46,7 +47,8 @@ class BinaryHolder {
   };
 
   BinaryHolder(IntrusiveRefCntPtr<vfs::FileSystem> VFS,
-               BinaryHolder::Options Opts = {});
+               BinaryHolder::Options Opts = {},
+               std::shared_ptr<cas::ObjectStore> CAS = nullptr);
 
   // Forward declarations for friend declaration.
   class ObjectEntry;
@@ -67,6 +69,9 @@ class BinaryHolder {
     Error load(IntrusiveRefCntPtr<vfs::FileSystem> VFS, StringRef Filename,
                TimestampTy Timestamp, BinaryHolder::Options = {});
 
+    bool load(cas::ObjectStore &CAS, StringRef Filename,
+              BinaryHolder::Options = {});
+
     /// Access all owned ObjectFiles.
     std::vector<const object::ObjectFile *> getObjects() const;
 
@@ -150,6 +155,8 @@ class BinaryHolder {
   /// Virtual File System instance.
   IntrusiveRefCntPtr<vfs::FileSystem> VFS;
 
+  std::shared_ptr<cas::ObjectStore> CAS;
+
   Options Opts;
 };
 

llvm/tools/dsymutil/Options.td

@@ -218,3 +218,16 @@ def dsym_search_path: Separate<["-", "--"], "D">,
   MetaVarName<"<path>">,
   HelpText<"Specify a directory that contain dSYM files to search for.">,
   Group<grp_general>;
+
+def cas : Separate<["-", "--"], "cas">,
+  MetaVarName<"<path>">,
+  HelpText<"Specify CAS directory to load CAS references from.">,
+  Group<grp_general>;
+def cas_plugin_path : Separate<["-", "--"], "cas-plugin-path">,
+  MetaVarName<"<path>">,
+  HelpText<"Path to CAS plugin library if used">,
+  Group<grp_general>;
+def cas_plugin_option : Separate<["-", "--"], "cas-plugin-option">,
+  MetaVarName<"<Option>=<Value>">,
+  HelpText<"Plugin CAS options">,
+  Group<grp_general>;

llvm/tools/dsymutil/dsymutil.cpp

@@ -23,6 +23,9 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/CASConfiguration.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFVerifier.h"
@@ -120,6 +123,7 @@ struct DsymutilOptions {
   DWARFVerify Verify = DWARFVerify::Default;
   ReproducerMode ReproMode = ReproducerMode::GenerateOnCrash;
   dsymutil::LinkOptions LinkOpts;
+  cas::CASConfiguration CASOptions;
 };
 
 /// Return a list of input files. This function has logic for dealing with the
@@ -397,6 +401,17 @@ static Expected<DsymutilOptions> getOptions(opt::InputArgList &Args) {
   for (auto *SearchPath : Args.filtered(OPT_dsym_search_path))
     Options.LinkOpts.DSYMSearchPaths.push_back(SearchPath->getValue());
 
+  if (opt::Arg *CASPath = Args.getLastArg(OPT_cas))
+    Options.CASOptions.CASPath = CASPath->getValue();
+
+  if (opt::Arg *CASPluginPath = Args.getLastArg(OPT_cas_plugin_path))
+    Options.CASOptions.PluginPath = CASPluginPath->getValue();
+
+  for (auto CASPluginOpt : Args.getAllArgValues(OPT_cas_plugin_option)) {
+    auto [Name, Val] = StringRef(CASPluginOpt).split('=');
+    Options.CASOptions.PluginOptions.emplace_back(Name, Val);
+  }
+
   if (Error E = verifyOptions(Options))
     return std::move(E);
   return Options;
@@ -666,12 +681,24 @@ int dsymutil_main(int argc, char **argv, const llvm::ToolContext &) {
       return EXIT_FAILURE;
     }
 
+  // Create CAS. Only support unified database for now.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  if (!Options.CASOptions.CASPath.empty()) {
+    auto DB = Options.CASOptions.createDatabases();
+    if (!DB) {
+      WithColor::error() << "failed to open CAS: " << toString(DB.takeError())
+                         << "\n";
+      return EXIT_FAILURE;
+    }
+    CAS = std::move(DB->first);
+  }
+
   for (auto &InputFile : Options.InputFiles) {
     // Shared a single binary holder for all the link steps.
     BinaryHolder::Options BinOpts;
     BinOpts.Verbose = Options.LinkOpts.Verbose;
     BinOpts.Warn = !Options.NoObjectTimestamp;
-    BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts);
+    BinaryHolder BinHolder(Options.LinkOpts.VFS, BinOpts, CAS);
 
     // Dump the symbol table for each input file and requested arch
     if (Options.DumpStab) {