[clang][gmodules] Use CASID to encode splitDwarfFilename

Teach clang to encode CASID as splitDwarfFilename for gmodule when
clang caching is enabled. This allows the outputs from compiler do not
contain paths to the clang module files, thus allows distributed caching
without the need of a unified clang module cache directory path.

Author: cachemeifyoucan

clang/include/clang/Basic/ASTSourceDescriptor.h

@@ -30,21 +30,23 @@ class ASTSourceDescriptor {
   StringRef Path;
   StringRef ASTFile;
   ASTFileSignature Signature;
+  StringRef CASID;
   Module *ClangModule = nullptr;
 
 public:
   ASTSourceDescriptor() = default;
   ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile,
-                      ASTFileSignature Signature)
+                      ASTFileSignature Signature, StringRef CASID)
       : PCHModuleName(std::move(Name)), Path(std::move(Path)),
-        ASTFile(std::move(ASTFile)), Signature(Signature) {}
+        ASTFile(std::move(ASTFile)), Signature(Signature), CASID(CASID) {}
   ASTSourceDescriptor(Module &M);
 
   std::string getModuleName() const;
   StringRef getPath() const { return Path; }
   StringRef getASTFile() const { return ASTFile; }
   ASTFileSignature getSignature() const { return Signature; }
   Module *getModuleOrNull() const { return ClangModule; }
+  StringRef getCASID() const { return CASID; }
 };
 
 } // namespace clang

clang/include/clang/Basic/DiagnosticCASKinds.td

@@ -23,8 +23,10 @@ def err_cas_depscan_daemon_connection: Error<
 def err_cas_depscan_failed: Error<
   "CAS-based dependency scan failed: %0">, DefaultFatal;
 def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal;
-def err_cas_unloadable_module : Error<
-  "module file '%0' not found: unloadable module cache key %1: %2">, DefaultFatal;
+def err_cas_unloadable_module
+    : Error<"module file '%0' not found: unloadable %select{casid|module cache "
+            "key}1 %2: %3">,
+      DefaultFatal;
 def err_cas_missing_module : Error<
   "module file '%0' not found: missing module cache key %1: %2">, DefaultFatal;
 def err_cas_missing_root_id : Error<

clang/include/clang/Basic/Module.h

@@ -274,6 +274,9 @@ class alignas(8) Module {
   /// The \c ActionCache key for this module, if any.
   std::optional<std::string> ModuleCacheKey;
 
+  /// The \c CASID for the loaded module, if any.
+  std::optional<std::string> CASID;
+
   /// The top-level headers associated with this module.
   llvm::SmallSetVector<FileEntryRef, 2> TopHeaders;
 
@@ -773,6 +776,15 @@ class alignas(8) Module {
     getTopLevelModule()->ModuleCacheKey = std::move(Key);
   }
 
+  std::optional<StringRef> getCASID() const {
+    return getTopLevelModule()->CASID;
+  }
+
+  void setCASID(std::string ID) {
+    assert(!getCASID() || *getCASID() == ID);
+    getTopLevelModule()->CASID = std::move(ID);
+  }
+
   /// Retrieve the umbrella directory as written.
   std::optional<DirectoryName> getUmbrellaDirAsWritten() const {
     if (const auto *Dir = std::get_if<DirectoryEntryRef>(&Umbrella))

clang/include/clang/Frontend/CompilerInstance.h

@@ -1031,8 +1031,8 @@ class CompilerInstance : public ModuleLoader {
   /// "-fmodule-file-cache-key", or an imported pcm file. Used in diagnostics.
   ///
   /// \returns true on failure.
-  bool addCachedModuleFile(StringRef Path, StringRef CacheKey,
-                           StringRef Provider);
+  bool addCachedModuleFile(StringRef Path, StringRef CASID, StringRef Provider,
+                           bool IsKey);
 
   ModuleCache &getModuleCache() const { return *ModCache; }
 

clang/include/clang/Serialization/ASTReader.h

@@ -287,6 +287,14 @@ class ASTReaderListener {
     return false;
   }
 
+  /// Called for each module CASID.
+  ///
+  /// \returns true to indicate the key cannot be loaded.
+  virtual bool readModuleCASID(StringRef ModuleName, StringRef Filename,
+                               StringRef CASID) {
+    return false;
+  }
+
   /// Indicates that a particular module file extension has been read.
   virtual void readModuleFileExtension(
                  const ModuleFileExtensionMetadata &Metadata) {}
@@ -343,6 +351,8 @@ class ChainedASTReaderListener : public ASTReaderListener {
   bool readIncludeTreeID(StringRef ID, bool Complain) override;
   bool readModuleCacheKey(StringRef ModuleName, StringRef Filename,
                           StringRef CacheKey) override;
+  bool readModuleCASID(StringRef ModuleName, StringRef Filename,
+                       StringRef CASID) override;
   void readModuleFileExtension(
          const ModuleFileExtensionMetadata &Metadata) override;
 };

clang/include/clang/Serialization/InMemoryModuleCache.h

@@ -30,6 +30,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   struct PCM {
     std::unique_ptr<llvm::MemoryBuffer> Buffer;
 
+    std::string CASID;
+
     /// Track whether this PCM is known to be good (either built or
     /// successfully imported by a CompilerInstance/ASTReader using this
     /// cache).
@@ -38,6 +40,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
     PCM() = default;
     PCM(std::unique_ptr<llvm::MemoryBuffer> Buffer)
         : Buffer(std::move(Buffer)) {}
+
+    PCM(std::unique_ptr<llvm::MemoryBuffer> Buffer, llvm::StringRef CASID)
+        : Buffer(std::move(Buffer)), CASID(CASID.str()) {}
   };
 
   /// Cache of buffers.
@@ -64,7 +69,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   /// \post state is Tentative
   /// \return a reference to the buffer as a convenience.
   llvm::MemoryBuffer &addPCM(llvm::StringRef Filename,
-                             std::unique_ptr<llvm::MemoryBuffer> Buffer);
+                             std::unique_ptr<llvm::MemoryBuffer> Buffer,
+                             llvm::StringRef CASID = "");
 
   /// Store a just-built PCM under the Filename.
   ///
@@ -90,6 +96,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase<InMemoryModuleCache> {
   /// Get a pointer to the pCM if it exists; else nullptr.
   llvm::MemoryBuffer *lookupPCM(llvm::StringRef Filename) const;
 
+  /// Get the PCM if it exits; else nullptr.
+  const PCM *lookup(llvm::StringRef Filename) const;
+
   /// Check whether the PCM is final and has been shown to work.
   ///
   /// \return true iff state is Final.

clang/include/clang/Serialization/ModuleFile.h

@@ -147,6 +147,9 @@ class ModuleFile {
   /// The \c ActionCache key for this module, or empty.
   std::string ModuleCacheKey;
 
+  /// The \c CASID for the module, or empty.
+  std::string CASID;
+
   /// The CAS filesystem root ID for implicit modules built with the dependency
   /// scanner, or empty.
   std::string CASFileSystemRootID;

clang/lib/Basic/ASTSourceDescriptor.cpp

@@ -21,6 +21,8 @@ ASTSourceDescriptor::ASTSourceDescriptor(Module &M)
     Path = M.Directory->getName();
   if (auto File = M.getASTFile())
     ASTFile = File->getName();
+  if (M.getCASID())
+    CASID = *M.getCASID();
 }
 
 std::string ASTSourceDescriptor::getModuleName() const {

clang/lib/CodeGen/CGDebugInfo.cpp

@@ -3401,6 +3401,11 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
         PCM = Mod.getPath();
     }
     llvm::sys::path::append(PCM, Mod.getASTFile());
+
+    // If module file has CASID, use CASID as PCM path.
+    if (!Mod.getCASID().empty())
+      PCM = Mod.getCASID();
+
     DIB.createCompileUnit(
         TheCU->getSourceLanguage(),
         // TODO: Support "Source" from external AST providers?

clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp

@@ -191,8 +191,8 @@ class PCHContainerGenerator : public ASTConsumer {
     // Prepare CGDebugInfo to emit debug info for a clang module.
     auto *DI = Builder->getModuleDebugInfo();
     StringRef ModuleName = llvm::sys::path::filename(MainFileName);
-    DI->setPCHDescriptor(
-        {ModuleName, "", OutputFileName, ASTFileSignature::createDISentinel()});
+    DI->setPCHDescriptor({ModuleName, "", OutputFileName,
+                          ASTFileSignature::createDISentinel(), /*CASID=*/""});
     DI->setModuleMap(MMap);
   }