[lldb] Teach LLDB to load gmodules from CAS

Teach lldb to load clang modules when gmodule + clang caching is used.

Author: cachemeifyoucan

lldb/include/lldb/Core/ModuleList.h

@@ -22,6 +22,7 @@
 #include "lldb/lldb-types.h"
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/CAS/CASConfiguration.h"
 #include "llvm/Support/RWMutex.h"
 
 #include <functional>
@@ -100,6 +101,17 @@ class ModuleListProperties : public Properties {
   bool GetSwiftEnableASTContext() const;
   // END SWIFT
 
+  // START CAS
+  /// Get CASPath set via properties.
+  FileSpec GetCASOnDiskPath() const;
+
+  /// Get CASPluginPath set via properties.
+  FileSpec GetCASPluginPath() const;
+
+  /// Get CASPluginOptions set via properties.
+  std::vector<std::pair<std::string, std::string>> GetCASPluginOptions() const;
+  // END CAS
+
   FileSpec GetClangModulesCachePath() const;
   bool SetClangModulesCachePath(const FileSpec &path);
   bool GetEnableExternalLookup() const;
@@ -522,6 +534,27 @@ class ModuleList {
                   llvm::SmallVectorImpl<lldb::ModuleSP> *old_modules,
                   bool *did_create_ptr, bool always_create = false);
 
+  // START CAS
+
+  /// Get CAS configuration using global module properties or from candidate
+  /// search path.
+  static std::optional<llvm::cas::CASConfiguration>
+  GetCASConfiguration(FileSpec CandidateConfigSearchPath);
+
+  /// Gets the shared module from CAS. It works the same as `GetSharedModule`
+  /// but the lookup is done inside the CAS.
+  ///
+  /// \return
+  ///    true if module is successfully loaded, false if module is not found
+  ///    in the CAS, error if there are any errors happened during the loading
+  ///    process.
+  static llvm::Expected<bool> GetSharedModuleFromCAS(ConstString module_name,
+                                                     llvm::StringRef cas_id,
+                                                     FileSpec cu_path,
+                                                     ModuleSpec &module_spec,
+                                                     lldb::ModuleSP &module_sp);
+  // END CAS
+
   static bool RemoveSharedModule(lldb::ModuleSP &module_sp);
 
   static void FindSharedModules(const ModuleSpec &module_spec,

lldb/scripts/clang-explicit-module-build.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""
+Usage: clang-explicit-module-build.py [-cas-path CAS_PATH] -- CLANG_COMMAND
+
+The script builds with clang explicit module. If `-cas-path` is used, clang
+explicit module build is going to enable compilation caching.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+
+def main():
+    argv = sys.argv[1:]
+    if "--" not in argv:
+        print("missing clang command")
+        exit(1)
+    dash_idx = argv.index("--")
+    clang_args = argv[dash_idx + 1:]
+    if len(clang_args) == 0:
+        print("empty clang command")
+        exit(1)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-cas-path", required=False)
+    args = parser.parse_args(argv[:dash_idx])
+
+    clang_exe = clang_args[0]
+    clang_scan_deps = os.path.join(
+        os.path.dirname(clang_exe), "clang-scan-deps")
+
+    scan_cmd = [clang_scan_deps]
+    if args.cas_path is None:
+        scan_cmd.extend(["-format", "experimental-full", "-o", "-", "--"])
+    else:
+        scan_cmd.extend(["-format", "experimental-include-tree-full",
+                        "-cas-path", args.cas_path, "-o", "-", "--"])
+    scan_cmd.extend(clang_args)
+    scan_result = json.loads(subprocess.check_output(scan_cmd))
+
+    # build module: assuming modules in reverse dependency order.
+    for module in reversed(scan_result["modules"]):
+        cmd = [clang_exe] + module["command-line"]
+        print(*cmd)
+        subprocess.check_call(cmd)
+
+    # build tu: assuming only one TU.
+    tu_cmd = [clang_exe] + \
+        scan_result["translation-units"][0]["commands"][0]["command-line"]
+    print(*tu_cmd)
+    subprocess.check_call(tu_cmd)
+
+
+if __name__ == "__main__":
+    main()

lldb/source/Core/CoreProperties.td

@@ -74,6 +74,19 @@ let Definition = "modulelist" in {
     DefaultTrue,
     Desc<"Enable instantiating Swift AST contexts.">;
 // END SWIFT
+// BEGIN CAS
+  def CASOnDiskPath: Property<"cas-path", "FileSpec">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"The path for CAS storage">;
+  def CASPluginPath: Property<"cas-plugin-path", "FileSpec">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"The path for CAS plugin dynamic library">;
+  def CASPluginOptions: Property<"cas-plugin-options", "Array">,
+    ElementType<"String">,
+    Desc<"A list of options to be passed to CAS plugins">;
+// END CAS
   def SymLinkPaths: Property<"debug-info-symlink-paths", "FileSpecList">,
     Global,
     DefaultStringValue<"">,

lldb/source/Core/ModuleList.cpp

@@ -21,18 +21,23 @@
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/DataBufferLLVM.h"
 #include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-defines.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/FileUtilities.h"
 
 #if defined(_WIN32)
 #include "lldb/Host/windows/PosixApi.h"
 #endif
 
 #include "clang/Driver/Driver.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/CASConfiguration.h"
+#include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Threading.h"
 #include "llvm/Support/raw_ostream.h"
@@ -282,6 +287,32 @@ bool ModuleListProperties::GetSwiftEnableASTContext() const {
 }
 // END SWIFT
 
+// START CAS
+FileSpec ModuleListProperties::GetCASOnDiskPath() const {
+  const uint32_t idx = ePropertyCASOnDiskPath;
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
+}
+
+FileSpec ModuleListProperties::GetCASPluginPath() const {
+  const uint32_t idx = ePropertyCASPluginPath;
+  return GetPropertyAtIndexAs<FileSpec>(idx, {});
+}
+
+std::vector<std::pair<std::string, std::string>>
+ModuleListProperties::GetCASPluginOptions() const {
+  Args args;
+  const uint32_t idx = ePropertyCASPluginOptions;
+  m_collection_sp->GetPropertyAtIndexAsArgs(idx, args);
+  std::vector<std::pair<std::string, std::string>> options;
+  for (auto &arg : args) {
+    llvm::StringRef opt = arg.c_str();
+    auto splitted = opt.split("=");
+    options.emplace_back(splitted.first.str(), splitted.second.str());
+  }
+  return options;
+}
+// END CAS
+
 FileSpec ModuleListProperties::GetLLDBIndexCachePath() const {
   const uint32_t idx = ePropertyLLDBIndexCachePath;
   return GetPropertyAtIndexAs<FileSpec>(idx, {});
@@ -1257,8 +1288,11 @@ class SharedModuleList {
 struct SharedModuleListInfo {
   SharedModuleList module_list;
   ModuleListProperties module_list_properties;
+  std::shared_ptr<llvm::cas::ObjectStore> cas_object_store;
+  std::mutex shared_lock;
 };
 }
+
 static SharedModuleListInfo &GetSharedModuleListInfo()
 {
   static SharedModuleListInfo *g_shared_module_list_info = nullptr;
@@ -1277,6 +1311,47 @@ static SharedModuleList &GetSharedModuleList() {
   return GetSharedModuleListInfo().module_list;
 }
 
+std::optional<llvm::cas::CASConfiguration>
+ModuleList::GetCASConfiguration(FileSpec CandidateConfigSearchPath) {
+  // Config CAS from properties.
+  llvm::cas::CASConfiguration cas_config;
+  cas_config.CASPath =
+      ModuleList::GetGlobalModuleListProperties().GetCASOnDiskPath().GetPath();
+  cas_config.PluginPath =
+      ModuleList::GetGlobalModuleListProperties().GetCASPluginPath().GetPath();
+  cas_config.PluginOptions =
+      ModuleList::GetGlobalModuleListProperties().GetCASPluginOptions();
+
+  if (!cas_config.CASPath.empty())
+    return cas_config;
+
+  auto search_config = llvm::cas::CASConfiguration::createFromSearchConfigFile(
+      CandidateConfigSearchPath.GetPath());
+  if (search_config)
+    return search_config->second;
+
+  return std::nullopt;
+}
+
+static llvm::Expected<std::shared_ptr<llvm::cas::ObjectStore>>
+GetOrCreateCASStorage(FileSpec CandidateConfigSearchPath) {
+  auto &shared_module_list = GetSharedModuleListInfo();
+  if (shared_module_list.cas_object_store)
+    return shared_module_list.cas_object_store;
+
+  auto config = ModuleList::GetCASConfiguration(CandidateConfigSearchPath);
+  if (!config)
+    return nullptr;
+
+  auto cas = config->createDatabases();
+  if (!cas)
+    return cas.takeError();
+
+  std::scoped_lock<std::mutex> lock(shared_module_list.shared_lock);
+  shared_module_list.cas_object_store = std::move(cas->first);
+  return shared_module_list.cas_object_store;
+}
+
 ModuleListProperties &ModuleList::GetGlobalModuleListProperties() {
   return GetSharedModuleListInfo().module_list_properties;
 }
@@ -1548,6 +1623,65 @@ ModuleList::GetSharedModule(const ModuleSpec &module_spec, ModuleSP &module_sp,
   return error;
 }
 
+static llvm::Expected<bool> loadModuleFromCAS(ConstString module_name,
+                                              llvm::StringRef cas_id,
+                                              FileSpec cu_path,
+                                              ModuleSpec &module_spec) {
+  auto maybe_cas = GetOrCreateCASStorage(cu_path);
+  if (!maybe_cas)
+    return maybe_cas.takeError();
+
+  auto cas = std::move(*maybe_cas);
+  if (!cas) {
+    LLDB_LOG(GetLog(LLDBLog::Modules),
+             "skip loading module '{0}' from CAS: CAS is not available",
+             module_name);
+    return false;
+  }
+
+  auto id = cas->parseID(cas_id);
+  if (!id) {
+    LLDB_LOG(GetLog(LLDBLog::Modules), "'{0}' is not valid CASID: {1}", cas_id,
+             toString(id.takeError()));
+    return false;
+  }
+
+  auto module_proxy = cas->getProxy(*id);
+  if (!module_proxy)
+    return module_proxy.takeError();
+
+  auto file_buffer =
+      std::make_shared<DataBufferLLVM>(module_proxy->getMemoryBuffer());
+
+  // Swap out the module_spec with the one loaded via CAS.
+  ModuleSpec loaded(module_spec.GetFileSpec(), module_spec.GetUUID(),
+                    std::move(file_buffer));
+  loaded.GetArchitecture() = module_spec.GetArchitecture();
+  module_spec = loaded;
+
+  LLDB_LOG(GetLog(LLDBLog::Modules), "loading module '{0}' using CASID '{1}'",
+           module_name, cas_id);
+  return true;
+}
+
+llvm::Expected<bool> ModuleList::GetSharedModuleFromCAS(
+    ConstString module_name, llvm::StringRef cas_id, FileSpec cu_path,
+    ModuleSpec &module_spec, lldb::ModuleSP &module_sp) {
+  auto loaded = loadModuleFromCAS(module_name, cas_id, cu_path, module_spec);
+  if (!loaded)
+    return loaded.takeError();
+
+  if (!*loaded)
+    return false;
+
+  auto status =
+      GetSharedModule(module_spec, module_sp, nullptr, nullptr, nullptr,
+                      /*always_create=*/true);
+  if (status.Success())
+    return true;
+  return status.takeError();
+}
+
 bool ModuleList::RemoveSharedModule(lldb::ModuleSP &module_sp) {
   return GetSharedModuleList().Remove(module_sp);
 }

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

@@ -2004,6 +2004,23 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() {
       continue;
 
     ModuleSpec dwo_module_spec;
+
+    dwo_module_spec.GetArchitecture() =
+        m_objfile_sp->GetModule()->GetArchitecture();
+
+    // Try load from CAS, if loaded, continue to next one.
+    auto loaded = ModuleList::GetSharedModuleFromCAS(
+        const_name, dwo_path, GetObjectFile()->GetFileSpec(), dwo_module_spec,
+        module_sp);
+    if (!loaded)
+      GetObjectFile()->GetModule()->ReportWarning(
+          "Failed to load module '{0}' from CAS: {1}", const_name,
+          toString(loaded.takeError()));
+
+    // succeed, loaded next module.
+    if (*loaded)
+      continue;
+
     dwo_module_spec.GetFileSpec().SetFile(dwo_path, FileSpec::Style::native);
     if (dwo_module_spec.GetFileSpec().IsRelative()) {
       const char *comp_dir =
@@ -2015,8 +2032,6 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() {
         dwo_module_spec.GetFileSpec().AppendPathComponent(dwo_path);
       }
     }
-    dwo_module_spec.GetArchitecture() =
-        m_objfile_sp->GetModule()->GetArchitecture();
 
     // When LLDB loads "external" modules it looks at the presence of
     // DW_AT_dwo_name. However, when the already created module

lldb/test/CMakeLists.txt

@@ -162,6 +162,8 @@ if(TARGET clang)
 
   # TestFullLtoStepping depends on LTO, and only runs when the compiler is clang.
   add_lldb_test_dependency(LTO)
+  # Clang explicit module build test requires clang-scan-deps
+  add_lldb_test_dependency(clang-scan-deps)
 
   if (TARGET libcxx OR ("libcxx" IN_LIST LLVM_ENABLE_RUNTIMES))
     set(LLDB_HAS_LIBCXX ON)