[Dependency Scanning] Re-use cached queried Clang dependency info in subsequent scanning stages

include/swift/AST/ModuleDependencies.h

@@ -277,10 +277,9 @@ class ModuleDependencyInfoStorageBase {
   /// The macro dependencies.
   std::map<std::string, MacroPluginDependency> macroDependencies;
 
-  /// A list of Clang modules that are visible to this Swift module. This
-  /// includes both direct Clang modules as well as transitive Clang
-  /// module dependencies when they are exported
-  llvm::StringSet<> visibleClangModules;
+  /// A list of Clang modules that are visible to this Swift module
+  /// as re-exported modular includes of its bridging header.
+  std::vector<std::string> bridgingHeaderVisibleClangModules;
 
   /// ModuleDependencyInfo is finalized (with all transitive dependencies
   /// and inputs).
@@ -893,14 +892,12 @@ class ModuleDependencyInfo {
       llvm_unreachable("Unexpected module dependency kind");
   }
 
-  llvm::StringSet<> &getVisibleClangModules() const {
-    return storage->visibleClangModules;
+  ArrayRef<std::string> getHeaderVisibleClangModules() const {
+    return storage->bridgingHeaderVisibleClangModules;
   }
-
-  void
-  addVisibleClangModules(const std::vector<std::string> &moduleNames) const {
-    storage->visibleClangModules.insert(moduleNames.begin(),
-                                        moduleNames.end());
+  void setHeaderVisibleClangModules(
+      const std::vector<std::string> &moduleNames) const {
+    storage->bridgingHeaderVisibleClangModules = moduleNames;
   }
 
   /// Whether explicit input paths of all the module dependencies
@@ -1073,6 +1070,9 @@ class ModuleDependenciesCache {
 private:
   /// Discovered dependencies
   ModuleDependenciesKindMap ModuleDependenciesMap;
+  /// A map from Clang module name to all visible modules to a client
+  /// of a by-name import of this Clang module
+  llvm::StringMap<std::vector<std::string>> clangModulesVisibleFromNamedLookup;
   /// Set containing all of the Clang modules that have already been seen.
   llvm::DenseSet<clang::tooling::dependencies::ModuleID> alreadySeenClangModules;
   /// Name of the module under scan
@@ -1111,6 +1111,8 @@ class ModuleDependenciesCache {
   bool hasDependency(StringRef moduleName) const;
   /// Whether we have cached dependency information for the given Swift module.
   bool hasSwiftDependency(StringRef moduleName) const;
+  /// Whether we have cached dependency information for the given Clang module.
+  bool hasClangDependency(StringRef moduleName) const;
   /// Report the number of recorded Clang dependencies
   int numberOfClangDependencies() const;
   /// Report the number of recorded Swift dependencies
@@ -1152,9 +1154,20 @@ class ModuleDependenciesCache {
   /// Query all cross-import overlay dependencies
   llvm::ArrayRef<ModuleDependencyID>
   getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const;
+
   /// Query all visible Clang modules for a given Swift dependency
-  llvm::StringSet<>&
-  getVisibleClangModules(ModuleDependencyID moduleID) const;
+  llvm::StringSet<>
+  getAllVisibleClangModules(ModuleDependencyID moduleID) const;
+  /// Query all Clang modules visible via a by-name lookup of given
+  /// Clang dependency
+  llvm::ArrayRef<std::string>
+  getVisibleClangModulesFromLookup(StringRef moduleName) const;
+  bool hasVisibleClangModulesFromLookup(StringRef moduleName) const;
+  /// Query all Clang modules visible from a given Swift module's
+  /// bridging header
+  llvm::ArrayRef<std::string>
+  getVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const;
+  bool hasVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const;
 
   /// Look for module dependencies for a module with the given ID
   ///
@@ -1230,8 +1243,8 @@ class ModuleDependenciesCache {
     const ModuleDependencyIDCollectionView dependencyIDs);
   /// Add to this module's set of visible Clang modules
   void
-  addVisibleClangModules(ModuleDependencyID moduleID,
-                         const std::vector<std::string> &moduleNames);
+  setVisibleClangModulesFromLookup(ModuleDependencyID clangModuleID,
+                                   const std::vector<std::string> &moduleNames);
 
   StringRef getMainModuleName() const { return mainScanModuleName; }
 

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -37,6 +37,14 @@ using ImportStatementInfoMap =
     std::unordered_map<ModuleDependencyID,
                        std::vector<ScannerImportStatementInfo>>;
 
+/// A map from a module ID to a collection of module IDs.
+using ModuleIDToModuleIDSetVectorMap =
+    std::unordered_map<ModuleDependencyID,
+                       ModuleDependencyIDSetVector>;
+
+using ModuleIDImportInfoPair =
+    std::pair<ModuleDependencyID, ScannerImportStatementInfo>;
+
 struct ScannerMetrics {
   /// Number of performed queries for a Swift dependency with a given name
   std::atomic<uint32_t> SwiftModuleQueries;
@@ -306,7 +314,7 @@ class ModuleDependencyScanner {
   void resolveSwiftModuleDependencies(
       const ModuleDependencyID &rootModuleID,
       ModuleDependencyIDSetVector &discoveredSwiftModules);
-  void resolveAllClangModuleDependencies(
+  void resolveClangModuleDependencies(
       ArrayRef<ModuleDependencyID> swiftModules,
       ModuleDependencyIDSetVector &discoveredClangModules);
   void resolveHeaderDependencies(
@@ -390,25 +398,19 @@ class ModuleDependencyScanner {
   ///    in \c failedToResolveImports.
   /// 4. Update the set of resolved Clang dependencies for each Swift
   ///    module dependency in \c resolvedClangDependenciesMap.
-  void cacheComputedClangModuleLookupResults(
+  void processBatchClangModuleQueryResult(
       const BatchClangModuleLookupResult &lookupResult,
       const ImportStatementInfoMap &unresolvedImportsMap,
       const ImportStatementInfoMap &unresolvedOptionalImportsMap,
-      ArrayRef<ModuleDependencyID> swiftModuleDependents,
       ModuleDependencyIDSetVector &allDiscoveredClangModules,
-      std::vector<std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
-          &failedToResolveImports,
-      std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-          &resolvedClangDependenciesMap);
+      std::vector<ModuleIDImportInfoPair> &failedToResolveImports,
+      ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap);
 
   /// Re-query some failed-to-resolve Clang imports from cache
   /// in chance they were brought in as transitive dependencies.
   void reQueryMissedModulesFromCache(
-      const std::vector<
-          std::pair<ModuleDependencyID, ScannerImportStatementInfo>>
-          &failedToResolveImports,
-      std::unordered_map<ModuleDependencyID, ModuleDependencyIDSetVector>
-          &resolvedClangDependenciesMap);
+      const std::vector<ModuleIDImportInfoPair> &failedToResolveImports,
+      ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap);
 
   /// Assuming the \c `moduleImport` failed to resolve,
   /// iterate over all binary Swift module dependencies with serialized

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -41,7 +41,7 @@ using llvm::BCVBR;
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 10;
 /// Increment this on every change.
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 4;
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 5;
 
 /// Various identifiers in this format will rely on having their strings mapped
 /// using this ID.
@@ -81,6 +81,7 @@ using FileIDArrayIDField = IdentifierIDField;
 using ContextHashIDField = IdentifierIDField;
 using ModuleCacheKeyIDField = IdentifierIDField;
 using ImportArrayIDField = IdentifierIDField;
+using VisibleModulesArrayIDField = IdentifierIDField;
 using LinkLibrariesArrayIDField = IdentifierIDField;
 using MacroDependenciesArrayIDField = IdentifierIDField;
 using SearchPathArrayIDField = IdentifierIDField;
@@ -109,6 +110,7 @@ enum {
   MACRO_DEPENDENCY_ARRAY_NODE,
   SEARCH_PATH_NODE,
   SEARCH_PATH_ARRAY_NODE,
+  VISIBLE_MODULES_NODE,
   IMPORT_STATEMENT_NODE,
   IMPORT_STATEMENT_ARRAY_NODE,
   OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE,
@@ -187,6 +189,14 @@ using SearchPathLayout =
 using SearchPathArrayLayout =
     BCRecordLayout<SEARCH_PATH_ARRAY_NODE, IdentifierIDArryField>;
 
+// A record capturing information about all Clang modules visible
+// from a given named Clang module dependency query
+using VisibleModulesLayout =
+  BCRecordLayout<VISIBLE_MODULES_NODE,        // ID
+                 IdentifierIDField,           // moduleIdentifier
+                 VisibleModulesArrayIDField   // visibleModulesArrayIdentifier
+                 >;
+
 // A record capturing information about a given 'import' statement
 // captured in a dependency node, including its source location.
 using ImportStatementLayout =
@@ -199,6 +209,7 @@ using ImportStatementLayout =
                    IsExportedImport,             // isExported
                    AccessLevelField              // accessLevel
                    >;
+
 using ImportStatementArrayLayout =
     BCRecordLayout<IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;
 using OptionalImportStatementArrayLayout =

lib/AST/ModuleDependencies.cpp

@@ -759,6 +759,9 @@ bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const {
 bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const {
   return findSwiftDependency(moduleName).has_value();
 }
+bool ModuleDependenciesCache::hasClangDependency(StringRef moduleName) const {
+  return findDependency(moduleName, ModuleDependencyKind::Clang).has_value();
+}
 
 int ModuleDependenciesCache::numberOfClangDependencies() const {
   return ModuleDependenciesMap.at(ModuleDependencyKind::Clang).size();
@@ -844,6 +847,16 @@ void ModuleDependenciesCache::recordClangDependency(
   }
 }
 
+void ModuleDependenciesCache::setVisibleClangModulesFromLookup(
+    ModuleDependencyID moduleID,
+    const std::vector<std::string> &visibleModules) {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::Clang);
+  if (visibleModules.empty())
+    return;
+
+  clangModulesVisibleFromNamedLookup[moduleID.ModuleName] = visibleModules;
+}
+
 void ModuleDependenciesCache::updateDependency(
     ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) {
   auto &map = getDependenciesMap(moduleID.Kind);
@@ -854,6 +867,10 @@ void ModuleDependenciesCache::updateDependency(
 void ModuleDependenciesCache::removeDependency(ModuleDependencyID moduleID) {
   auto &map = getDependenciesMap(moduleID.Kind);
   map.erase(moduleID.ModuleName);
+  // If we are removing a Clang module which was queried by-name
+  // in a prior scan, we must re-compute its set of visible modules.
+  if (moduleID.Kind == ModuleDependencyKind::Clang)
+    clangModulesVisibleFromNamedLookup.erase(moduleID.ModuleName);
 }
 
 void ModuleDependenciesCache::setImportedSwiftDependencies(
@@ -936,22 +953,52 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies(
       moduleInfo.getImportedClangDependencies());
 }
 
-void ModuleDependenciesCache::addVisibleClangModules(
-    ModuleDependencyID moduleID, const std::vector<std::string> &moduleNames) {
-  if (moduleNames.empty())
-    return;
-  auto dependencyInfo = findKnownDependency(moduleID);
-  auto updatedDependencyInfo = dependencyInfo;
-  updatedDependencyInfo.addVisibleClangModules(moduleNames);
-  updateDependency(moduleID, updatedDependencyInfo);
+llvm::StringSet<> ModuleDependenciesCache::getAllVisibleClangModules(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  llvm::StringSet<> result;
+  if (hasVisibleClangModulesViaHeader(moduleID)) {
+    auto headerVisibleModules = getVisibleClangModulesViaHeader(moduleID);
+    result.insert(headerVisibleModules.begin(), headerVisibleModules.end());
+  }
+  for (const auto &clangDepID :
+       findKnownDependency(moduleID).getImportedClangDependencies()) {
+    assert(hasVisibleClangModulesFromLookup(clangDepID.ModuleName));
+    auto visibleModulesViaImport =
+      getVisibleClangModulesFromLookup(clangDepID.ModuleName);
+    result.insert(visibleModulesViaImport.begin(),
+                  visibleModulesViaImport.end());
+  }
+  return result;
+}
+
+ArrayRef<std::string> ModuleDependenciesCache::getVisibleClangModulesFromLookup(
+    StringRef moduleName) const {
+  ASSERT(hasVisibleClangModulesFromLookup(moduleName));
+  return clangModulesVisibleFromNamedLookup.at(moduleName);
+}
+
+bool ModuleDependenciesCache::hasVisibleClangModulesFromLookup(
+    StringRef moduleName) const {
+  return clangModulesVisibleFromNamedLookup.contains(moduleName);
 }
 
-llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules(
+llvm::ArrayRef<std::string>
+ModuleDependenciesCache::getVisibleClangModulesViaHeader(
+    ModuleDependencyID moduleID) const {
+  ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
+         moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
+         moduleID.Kind == ModuleDependencyKind::SwiftBinary);
+  return findKnownDependency(moduleID).getHeaderVisibleClangModules();
+}
+bool ModuleDependenciesCache::hasVisibleClangModulesViaHeader(
     ModuleDependencyID moduleID) const {
   ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource ||
          moduleID.Kind == ModuleDependencyKind::SwiftInterface ||
          moduleID.Kind == ModuleDependencyKind::SwiftBinary);
-  return findKnownDependency(moduleID).getVisibleClangModules();
+  return !getVisibleClangModulesViaHeader(moduleID).empty();
 }
 
 ModuleDependencyIDCollectionView

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -411,6 +411,23 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
       break;
     }
 
+    case VISIBLE_MODULES_NODE: {
+      uint64_t identifierID;
+      uint64_t valueArrayID;
+      VisibleModulesLayout::readRecord(Scratch, identifierID, valueArrayID);
+      auto moduleName = getIdentifier(identifierID);
+      if (!moduleName)
+        llvm::report_fatal_error(
+            "Bad visible modules info: no module name");
+      auto values = getStringArray(valueArrayID);
+      if (!values)
+        llvm::report_fatal_error(
+            "Bad visible modules info: modules");
+
+      cache.clangModulesVisibleFromNamedLookup[moduleName.value()] = values.value();
+      break;
+    }
+
     case IMPORT_STATEMENT_NODE: {
       unsigned importIdentifierID, bufferIdentifierID;
       unsigned lineNumber, columnNumber;
@@ -1056,6 +1073,7 @@ enum ModuleIdentifierArrayKind : uint8_t {
   BridgingHeaderBuildCommandLine,
   NonPathCommandLine,
   FileDependencies,
+  VisibleClangModulesFromLookup,
   LastArrayKind
 };
 
@@ -1166,6 +1184,7 @@ class ModuleDependenciesCacheSerializer {
   unsigned writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo);
   void writeSearchPathsArray(unsigned startIndex, unsigned count);
 
+  void writeVisibleClangModuleInfo(const ModuleDependenciesCache &cache);
   void writeImportStatementInfos(const ModuleDependenciesCache &cache);
   unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo,
                                      bool optional);
@@ -1435,6 +1454,21 @@ void ModuleDependenciesCacheSerializer::writeSearchPathsArray(unsigned startInde
       Out, ScratchRecord, AbbrCodes[SearchPathArrayLayout::Code], vec);
 }
 
+void ModuleDependenciesCacheSerializer::writeVisibleClangModuleInfo(
+    const ModuleDependenciesCache &cache) {
+  using namespace graph_block;
+  for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) {
+    auto moduleID =
+        ModuleDependencyID{entry.getKey().str(), ModuleDependencyKind::Clang};
+    VisibleModulesLayout::emitRecord(
+        Out, ScratchRecord, AbbrCodes[VisibleModulesLayout::Code],
+        getIdentifier(moduleID.ModuleName),
+        getIdentifierArrayID(
+            moduleID,
+            ModuleIdentifierArrayKind::VisibleClangModulesFromLookup));
+  }
+}
+
 void ModuleDependenciesCacheSerializer::writeImportStatementInfos(
     const ModuleDependenciesCache &cache) {
   unsigned lastImportInfoIndex = 0;
@@ -1798,6 +1832,15 @@ unsigned ModuleDependenciesCacheSerializer::getOptionalImportStatementsArrayID(
 void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
     const ModuleDependenciesCache &cache) {
   addIdentifier(cache.scannerContextHash);
+
+  for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) {
+    auto moduleName = entry.getKey().str();
+    addIdentifier(moduleName);
+    addStringArray({moduleName, ModuleDependencyKind::Clang},
+                   ModuleIdentifierArrayKind::VisibleClangModulesFromLookup,
+                   entry.second);
+  }
+
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind) {
     auto modMap = cache.getDependenciesMap(kind);
@@ -1973,6 +2016,7 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   registerRecordAbbr<MacroDependencyArrayLayout>();
   registerRecordAbbr<SearchPathLayout>();
   registerRecordAbbr<SearchPathArrayLayout>();
+  registerRecordAbbr<VisibleModulesLayout>();
   registerRecordAbbr<ImportStatementLayout>();
   registerRecordAbbr<ImportStatementArrayLayout>();
   registerRecordAbbr<OptionalImportStatementArrayLayout>();
@@ -1998,6 +2042,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   // Write the arrays
   writeArraysOfIdentifiers();
 
+  // Write the cached sets of visible modules
+  writeVisibleClangModuleInfo(cache);
+
   // Write all the import statement info
   writeImportStatementInfos(cache);