From 07de483e30064002fdc89e84eef4fcd9bd89fdf4 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 18 Dec 2025 10:40:00 -0800 Subject: [PATCH 1/2] [Dependency Scanning] Re-use cached queried Clang dependency info in subsequent scanning stages Dependency Scanning is recursive over discovered Swift module overlays and cross-import overlays. In the main 'resolveImportedModuleDependencies' routine, after all Swift and Clang dependencies of the main module are discovered, the scanner looks up Swift overlays for discovered Clang modules. For each such Swift overlay, the scanner will then proceed to call 'resolveImportedModuleDependencies' on the overlay module. On these subsequent recursive calls to 'resolveImportedModuleDependencies', Clang dependency resolution will re-query all imports of the overlay module which do not resolve to Swift modules, even if they had been queried previously in a parent invocation. This change adds the ability to re-use previously-queried-and-cached Clang module dependency information by having the dependency cache also store the set of visible modules which resulted from each by-name lookup. --- include/swift/AST/ModuleDependencies.h | 43 ++-- .../DependencyScan/ModuleDependencyScanner.h | 26 +- lib/AST/ModuleDependencies.cpp | 67 +++++- .../ModuleDependencyScanner.cpp | 223 ++++++++++-------- lib/DependencyScan/ScanDependencies.cpp | 1 - .../basic_query_metrics.swift | 2 +- .../error_source_locations.swift | 19 +- .../module_deps_cache_reuse.swift | 4 +- 8 files changed, 241 insertions(+), 144 deletions(-) diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index f70410465355f..ed1a0443d31ee 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -277,10 +277,9 @@ class ModuleDependencyInfoStorageBase { /// The macro dependencies. std::map macroDependencies; - /// A list of Clang modules that are visible to this Swift module. This - /// includes both direct Clang modules as well as transitive Clang - /// module dependencies when they are exported - llvm::StringSet<> visibleClangModules; + /// A list of Clang modules that are visible to this Swift module + /// as re-exported modular includes of its bridging header. + std::vector bridgingHeaderVisibleClangModules; /// ModuleDependencyInfo is finalized (with all transitive dependencies /// and inputs). @@ -893,14 +892,12 @@ class ModuleDependencyInfo { llvm_unreachable("Unexpected module dependency kind"); } - llvm::StringSet<> &getVisibleClangModules() const { - return storage->visibleClangModules; + ArrayRef getHeaderVisibleClangModules() const { + return storage->bridgingHeaderVisibleClangModules; } - - void - addVisibleClangModules(const std::vector &moduleNames) const { - storage->visibleClangModules.insert(moduleNames.begin(), - moduleNames.end()); + void setHeaderVisibleClangModules( + const std::vector &moduleNames) const { + storage->bridgingHeaderVisibleClangModules = moduleNames; } /// Whether explicit input paths of all the module dependencies @@ -1073,6 +1070,9 @@ class ModuleDependenciesCache { private: /// Discovered dependencies ModuleDependenciesKindMap ModuleDependenciesMap; + /// A map from Clang module name to all visible modules to a client + /// of a by-name import of this Clang module + llvm::StringMap> clangModulesVisibleFromNamedLookup; /// Set containing all of the Clang modules that have already been seen. llvm::DenseSet alreadySeenClangModules; /// Name of the module under scan @@ -1111,6 +1111,8 @@ class ModuleDependenciesCache { bool hasDependency(StringRef moduleName) const; /// Whether we have cached dependency information for the given Swift module. bool hasSwiftDependency(StringRef moduleName) const; + /// Whether we have cached dependency information for the given Clang module. + bool hasClangDependency(StringRef moduleName) const; /// Report the number of recorded Clang dependencies int numberOfClangDependencies() const; /// Report the number of recorded Swift dependencies @@ -1152,9 +1154,20 @@ class ModuleDependenciesCache { /// Query all cross-import overlay dependencies llvm::ArrayRef getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const; + /// Query all visible Clang modules for a given Swift dependency - llvm::StringSet<>& - getVisibleClangModules(ModuleDependencyID moduleID) const; + llvm::StringSet<> + getAllVisibleClangModules(ModuleDependencyID moduleID) const; + /// Query all Clang modules visible via a by-name lookup of given + /// Clang dependency + llvm::ArrayRef + getVisibleClangModulesFromLookup(StringRef moduleName) const; + bool hasVisibleClangModulesFromLookup(StringRef moduleName) const; + /// Query all Clang modules visible from a given Swift module's + /// bridging header + llvm::ArrayRef + getVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const; + bool hasVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const; /// Look for module dependencies for a module with the given ID /// @@ -1230,8 +1243,8 @@ class ModuleDependenciesCache { const ModuleDependencyIDCollectionView dependencyIDs); /// Add to this module's set of visible Clang modules void - addVisibleClangModules(ModuleDependencyID moduleID, - const std::vector &moduleNames); + setVisibleClangModulesFromLookup(ModuleDependencyID clangModuleID, + const std::vector &moduleNames); StringRef getMainModuleName() const { return mainScanModuleName; } diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h index 3f803bd6817c5..1a1634172e4ac 100644 --- a/include/swift/DependencyScan/ModuleDependencyScanner.h +++ b/include/swift/DependencyScan/ModuleDependencyScanner.h @@ -37,6 +37,14 @@ using ImportStatementInfoMap = std::unordered_map>; +/// A map from a module ID to a collection of module IDs. +using ModuleIDToModuleIDSetVectorMap = + std::unordered_map; + +using ModuleIDImportInfoPair = + std::pair; + struct ScannerMetrics { /// Number of performed queries for a Swift dependency with a given name std::atomic SwiftModuleQueries; @@ -306,7 +314,7 @@ class ModuleDependencyScanner { void resolveSwiftModuleDependencies( const ModuleDependencyID &rootModuleID, ModuleDependencyIDSetVector &discoveredSwiftModules); - void resolveAllClangModuleDependencies( + void resolveClangModuleDependencies( ArrayRef swiftModules, ModuleDependencyIDSetVector &discoveredClangModules); void resolveHeaderDependencies( @@ -390,25 +398,19 @@ class ModuleDependencyScanner { /// in \c failedToResolveImports. /// 4. Update the set of resolved Clang dependencies for each Swift /// module dependency in \c resolvedClangDependenciesMap. - void cacheComputedClangModuleLookupResults( + void processBatchClangModuleQueryResult( const BatchClangModuleLookupResult &lookupResult, const ImportStatementInfoMap &unresolvedImportsMap, const ImportStatementInfoMap &unresolvedOptionalImportsMap, - ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules, - std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap); + std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap); /// Re-query some failed-to-resolve Clang imports from cache /// in chance they were brought in as transitive dependencies. void reQueryMissedModulesFromCache( - const std::vector< - std::pair> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap); + const std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap); /// Assuming the \c `moduleImport` failed to resolve, /// iterate over all binary Swift module dependencies with serialized diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 0e843a8651bd1..beb6040f437eb 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -759,6 +759,9 @@ bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const { bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const { return findSwiftDependency(moduleName).has_value(); } +bool ModuleDependenciesCache::hasClangDependency(StringRef moduleName) const { + return findDependency(moduleName, ModuleDependencyKind::Clang).has_value(); +} int ModuleDependenciesCache::numberOfClangDependencies() const { return ModuleDependenciesMap.at(ModuleDependencyKind::Clang).size(); @@ -844,6 +847,16 @@ void ModuleDependenciesCache::recordClangDependency( } } +void ModuleDependenciesCache::setVisibleClangModulesFromLookup( + ModuleDependencyID moduleID, + const std::vector &visibleModules) { + ASSERT(moduleID.Kind == ModuleDependencyKind::Clang); + if (visibleModules.empty()) + return; + + clangModulesVisibleFromNamedLookup[moduleID.ModuleName] = visibleModules; +} + void ModuleDependenciesCache::updateDependency( ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) { auto &map = getDependenciesMap(moduleID.Kind); @@ -854,6 +867,10 @@ void ModuleDependenciesCache::updateDependency( void ModuleDependenciesCache::removeDependency(ModuleDependencyID moduleID) { auto &map = getDependenciesMap(moduleID.Kind); map.erase(moduleID.ModuleName); + // If we are removing a Clang module which was queried by-name + // in a prior scan, we must re-compute its set of visible modules. + if (moduleID.Kind == ModuleDependencyKind::Clang) + clangModulesVisibleFromNamedLookup.erase(moduleID.ModuleName); } void ModuleDependenciesCache::setImportedSwiftDependencies( @@ -936,22 +953,52 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies( moduleInfo.getImportedClangDependencies()); } -void ModuleDependenciesCache::addVisibleClangModules( - ModuleDependencyID moduleID, const std::vector &moduleNames) { - if (moduleNames.empty()) - return; - auto dependencyInfo = findKnownDependency(moduleID); - auto updatedDependencyInfo = dependencyInfo; - updatedDependencyInfo.addVisibleClangModules(moduleNames); - updateDependency(moduleID, updatedDependencyInfo); +llvm::StringSet<> ModuleDependenciesCache::getAllVisibleClangModules( + ModuleDependencyID moduleID) const { + ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || + moduleID.Kind == ModuleDependencyKind::SwiftInterface || + moduleID.Kind == ModuleDependencyKind::SwiftBinary); + llvm::StringSet<> result; + if (hasVisibleClangModulesViaHeader(moduleID)) { + auto headerVisibleModules = getVisibleClangModulesViaHeader(moduleID); + result.insert(headerVisibleModules.begin(), headerVisibleModules.end()); + } + for (const auto &clangDepID : + findKnownDependency(moduleID).getImportedClangDependencies()) { + assert(hasVisibleClangModulesFromLookup(clangDepID.ModuleName)); + auto visibleModulesViaImport = + getVisibleClangModulesFromLookup(clangDepID.ModuleName); + result.insert(visibleModulesViaImport.begin(), + visibleModulesViaImport.end()); + } + return result; +} + +ArrayRef ModuleDependenciesCache::getVisibleClangModulesFromLookup( + StringRef moduleName) const { + ASSERT(hasVisibleClangModulesFromLookup(moduleName)); + return clangModulesVisibleFromNamedLookup.at(moduleName); +} + +bool ModuleDependenciesCache::hasVisibleClangModulesFromLookup( + StringRef moduleName) const { + return clangModulesVisibleFromNamedLookup.contains(moduleName); } -llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules( +llvm::ArrayRef +ModuleDependenciesCache::getVisibleClangModulesViaHeader( + ModuleDependencyID moduleID) const { + ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || + moduleID.Kind == ModuleDependencyKind::SwiftInterface || + moduleID.Kind == ModuleDependencyKind::SwiftBinary); + return findKnownDependency(moduleID).getHeaderVisibleClangModules(); +} +bool ModuleDependenciesCache::hasVisibleClangModulesViaHeader( ModuleDependencyID moduleID) const { ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || moduleID.Kind == ModuleDependencyKind::SwiftInterface || moduleID.Kind == ModuleDependencyKind::SwiftBinary); - return findKnownDependency(moduleID).getVisibleClangModules(); + return !getVisibleClangModulesViaHeader(moduleID).empty(); } ModuleDependencyIDCollectionView diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 6d492ae752e89..a4109a88b8253 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -878,7 +878,7 @@ ModuleDependencyScanner::performDependencyScan(ModuleDependencyID rootModuleID) // If scanning for an individual Clang module, simply resolve its imports if (rootModuleID.Kind == ModuleDependencyKind::Clang) { ModuleDependencyIDSetVector discoveredClangModules; - resolveAllClangModuleDependencies({}, discoveredClangModules); + resolveClangModuleDependencies({}, discoveredClangModules); return discoveredClangModules.takeVector(); } @@ -932,8 +932,8 @@ ModuleDependencyScanner::resolveImportedModuleDependencies( // This operation is done by gathering all unresolved import // identifiers and querying them in-parallel to the Clang // dependency scanner. - resolveAllClangModuleDependencies(discoveredSwiftModules.getArrayRef(), - allModules); + resolveClangModuleDependencies(discoveredSwiftModules.getArrayRef(), + allModules); // For each discovered Swift module which was built with a // bridging header, scan the header for module dependencies. @@ -984,13 +984,22 @@ void ModuleDependencyScanner::resolveSwiftModuleDependencies( return; } -static void -gatherUnresolvedImports(ModuleDependenciesCache &cache, - ASTContext &scanASTContext, - ArrayRef swiftModuleDependents, - ModuleDependencyIDSetVector &allDiscoveredClangModules, - ImportStatementInfoMap &unresolvedImportsMap, - ImportStatementInfoMap &unresolvedOptionalImportsMap) { +static void findAllReachableClangModules(ModuleDependencyID moduleID, + const ModuleDependenciesCache &cache, + ModuleDependencyIDSetVector &reachableClangModules) { + if (!reachableClangModules.insert(moduleID)) + return; + for (const auto &depID : cache.getImportedClangDependencies(moduleID)) + findAllReachableClangModules(depID, cache, reachableClangModules); +} + +static void gatherUnresolvedImports( + ModuleDependenciesCache &cache, ASTContext &scanASTContext, + ArrayRef swiftModuleDependents, + ModuleDependencyIDSetVector &allDiscoveredClangModules, + ImportStatementInfoMap &unresolvedImportsMap, + ImportStatementInfoMap &unresolvedOptionalImportsMap, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { for (const auto &moduleID : swiftModuleDependents) { auto moduleDependencyInfo = cache.findKnownDependency(moduleID); auto unresolvedImports = @@ -1002,34 +1011,49 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, .emplace(moduleID, std::vector()) .first->second; - // If we have already resolved Clang dependencies for this module, + // If we have already fully resolved Clang dependencies for this module, // then we have the entire dependency sub-graph already computed for // it and ready to be added to 'allDiscoveredClangModules' without // additional scanning. if (!moduleDependencyInfo.getImportedClangDependencies().empty()) { auto directClangDeps = cache.getImportedClangDependencies(moduleID); ModuleDependencyIDSetVector reachableClangModules; - reachableClangModules.insert(directClangDeps.begin(), - directClangDeps.end()); - for (unsigned currentModuleIdx = 0; - currentModuleIdx < reachableClangModules.size(); - ++currentModuleIdx) { - auto moduleID = reachableClangModules[currentModuleIdx]; - auto dependencies = - cache.findKnownDependency(moduleID).getImportedClangDependencies(); - reachableClangModules.insert(dependencies.begin(), dependencies.end()); - } - allDiscoveredClangModules.insert(reachableClangModules.begin(), - reachableClangModules.end()); + for (const auto &depID : directClangDeps) + findAllReachableClangModules(depID, cache, reachableClangModules); + allDiscoveredClangModules.insert( + reachableClangModules.getArrayRef().begin(), + reachableClangModules.getArrayRef().end()); continue; } else { - // We need to query the Clang dependency scanner for this module's - // non-Swift imports llvm::StringSet<> resolvedImportIdentifiers; + // Mark all import identifiers which were resolved to Swift dependencies + // as resolved for (const auto &resolvedDep : moduleDependencyInfo.getImportedSwiftDependencies()) resolvedImportIdentifiers.insert(resolvedDep.ModuleName); + // Mark all import identifiers which can be fully resolved to a cached + // Clang dependency (including visible modules) as resolved + auto markCachedClangDependenciesResolved = + [&cache, &moduleID, &resolvedImportIdentifiers, + &resolvedClangDependenciesMap]( + ArrayRef imports) { + for (const auto &import : imports) { + auto &importIdentifier = import.importIdentifier; + if (!resolvedImportIdentifiers.contains(importIdentifier) && + cache.hasClangDependency(importIdentifier) && + cache.hasVisibleClangModulesFromLookup(importIdentifier)) { + resolvedImportIdentifiers.insert(importIdentifier); + resolvedClangDependenciesMap[moduleID].insert( + {importIdentifier, ModuleDependencyKind::Clang}); + } + } + }; + markCachedClangDependenciesResolved( + moduleDependencyInfo.getModuleImports()); + markCachedClangDependenciesResolved( + moduleDependencyInfo.getOptionalModuleImports()); + // When querying a *clang* module 'CxxStdlib' we must // instead expect a module called 'std'... auto addCanonicalClangModuleImport = @@ -1047,6 +1071,8 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, } }; + // We need to query the Clang dependency scanner for all of this module's + // unresolved imports for (const auto &depImport : moduleDependencyInfo.getModuleImports()) if (!resolvedImportIdentifiers.contains(depImport.importIdentifier)) addCanonicalClangModuleImport(depImport, *unresolvedImports); @@ -1059,10 +1085,8 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, } void ModuleDependencyScanner::reQueryMissedModulesFromCache( - const std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap) { + const std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { // It is possible that a specific import resolution failed because we are // attempting to resolve a module which can only be brought in via a // modulemap of a different Clang module dependency which is not otherwise @@ -1096,8 +1120,9 @@ void ModuleDependencyScanner::reQueryMissedModulesFromCache( if (optionalCachedModuleInfo) { resolvedClangDependenciesMap[unresolvedImport.first].insert( unresolvedModuleID); - DependencyCache.addVisibleClangModules( - unresolvedImport.first, {unresolvedImport.second.importIdentifier}); + DependencyCache.setVisibleClangModulesFromLookup( + unresolvedModuleID, + {unresolvedImport.second.importIdentifier}); ScanDiagnosticReporter.registerNamedClangDependency(); } else { // Failed to resolve module dependency. @@ -1156,61 +1181,76 @@ void ModuleDependencyScanner::performClangModuleLookup( ScanningThreadPool.wait(); } -void ModuleDependencyScanner::cacheComputedClangModuleLookupResults( +void ModuleDependencyScanner::processBatchClangModuleQueryResult( const BatchClangModuleLookupResult &lookupResult, const ImportStatementInfoMap &unresolvedImportsMap, const ImportStatementInfoMap &unresolvedOptionalImportsMap, - ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules, - std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap) { - for (const auto &moduleID : swiftModuleDependents) { - auto recordResolvedClangModuleImport = - [this, &lookupResult, &resolvedClangDependenciesMap, - &allDiscoveredClangModules, moduleID, &failedToResolveImports]( - const ScannerImportStatementInfo &moduleImport, - bool optionalImport) { - auto &moduleIdentifier = moduleImport.importIdentifier; - auto dependencyID = - ModuleDependencyID{moduleIdentifier, ModuleDependencyKind::Clang}; - - // Add visible Clang modules for this query to the depending - // Swift module - if (lookupResult.visibleModules.contains(moduleIdentifier)) - DependencyCache.addVisibleClangModules( - moduleID, lookupResult.visibleModules.at(moduleIdentifier)); - - // Add the resolved dependency ID + std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { + + llvm::StringSet<> cachedNamedDependencies; + auto recordResolvedClangModuleImport = + [this, &lookupResult, &resolvedClangDependenciesMap, + &allDiscoveredClangModules, &failedToResolveImports, + &cachedNamedDependencies]( + const ModuleDependencyID &moduleID, + const ScannerImportStatementInfo &moduleImport, bool optionalImport) { + auto &moduleIdentifier = moduleImport.importIdentifier; + auto dependencyID = + ModuleDependencyID{moduleIdentifier, ModuleDependencyKind::Clang}; + + // A successful named query will have returned a set of visible modules + if (lookupResult.visibleModules.contains(moduleIdentifier)) { + // If this dependency has already been recorded in the cache, + // mark it as resolved for the current dependent and exit. + if (!cachedNamedDependencies.insert(moduleIdentifier).second) { + resolvedClangDependenciesMap[moduleID].insert(dependencyID); + return; + } + + ScanDiagnosticReporter.registerNamedClangDependency(); + DependencyCache.setVisibleClangModulesFromLookup( + dependencyID, lookupResult.visibleModules.at(moduleIdentifier)); + resolvedClangDependenciesMap[moduleID].insert(dependencyID); + + // If this module was not seen before as a transitive dependency + // of another lookup, record it into the cache if (lookupResult.discoveredDependencyInfos.contains( moduleIdentifier)) { - if (!DependencyCache.hasDependency(dependencyID)) - ScanDiagnosticReporter.registerNamedClangDependency(); - auto dependencyInfo = lookupResult.discoveredDependencyInfos.at( - moduleImport.importIdentifier); allDiscoveredClangModules.insert(dependencyID); DependencyCache.recordClangDependency( - dependencyInfo, ScanASTContext.Diags, [this](auto &clangDep) { + lookupResult.discoveredDependencyInfos.at( + moduleImport.importIdentifier), + ScanASTContext.Diags, [this](auto &clangDep) { return bridgeClangModuleDependency(clangDep); }); - resolvedClangDependenciesMap[moduleID].insert(dependencyID); - } else if (!optionalImport) { - // Otherwise, we failed to resolve this dependency. We will try - // again using the cache after all other imports have been - // resolved. If that fails too, a scanning failure will be - // diagnosed. - failedToResolveImports.push_back( - std::make_pair(moduleID, moduleImport)); + } else { + // If the query produced a set of visible modules but not + // a `ModuleDeps` info for the queried module itself, then + // it has to have been included in the set of already-seen + // module dependencies from a prior query. + assert(DependencyCache.hasDependency(dependencyID)); } - }; + } else if (!optionalImport) { + // Otherwise, we failed to resolve this dependency. We will try + // again using the cache after all other imports have been + // resolved. If that fails too, a scanning failure will be + // diagnosed. + failedToResolveImports.push_back( + std::make_pair(moduleID, moduleImport)); + } + }; - for (const auto &unresolvedImport : unresolvedImportsMap.at(moduleID)) - recordResolvedClangModuleImport(unresolvedImport, false); - for (const auto &unresolvedImport : - unresolvedOptionalImportsMap.at(moduleID)) - recordResolvedClangModuleImport(unresolvedImport, true); - } + for (const auto &moduleUnresolvedImports : unresolvedImportsMap) + for (const auto &unresolvedImport : moduleUnresolvedImports.second) + recordResolvedClangModuleImport(moduleUnresolvedImports.first, + unresolvedImport, false); + + for (const auto &moduleUnresolvedImports : unresolvedOptionalImportsMap) + for (const auto &unresolvedImport : moduleUnresolvedImports.second) + recordResolvedClangModuleImport(moduleUnresolvedImports.first, + unresolvedImport, true); // Use the computed scan results to record all transitive clang module // dependencies @@ -1227,33 +1267,32 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults( } } -void ModuleDependencyScanner::resolveAllClangModuleDependencies( +void ModuleDependencyScanner::resolveClangModuleDependencies( ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules) { - // Gather all unresolved imports which must correspond to + // Gather all remaining unresolved imports which must correspond to // Clang modules (since no Swift module for them was found). ImportStatementInfoMap unresolvedImportsMap; ImportStatementInfoMap unresolvedOptionalImportsMap; - gatherUnresolvedImports(DependencyCache, ScanASTContext, swiftModuleDependents, - allDiscoveredClangModules, unresolvedImportsMap, - unresolvedOptionalImportsMap); + ModuleIDToModuleIDSetVectorMap resolvedClangDependenciesMap; + gatherUnresolvedImports(DependencyCache, ScanASTContext, + swiftModuleDependents, allDiscoveredClangModules, + unresolvedImportsMap, unresolvedOptionalImportsMap, + resolvedClangDependenciesMap); // Execute parallel lookup of all unresolved import // identifiers as Clang modules. BatchClangModuleLookupResult lookupResult; - performClangModuleLookup( - unresolvedImportsMap, unresolvedOptionalImportsMap, lookupResult); + performClangModuleLookup(unresolvedImportsMap, unresolvedOptionalImportsMap, + lookupResult); // Use the computed scan results to record directly-queried clang module // dependencies. - std::vector> - failedToResolveImports; - std::unordered_map - resolvedClangDependenciesMap; - cacheComputedClangModuleLookupResults( + std::vector failedToResolveImports; + processBatchClangModuleQueryResult( lookupResult, unresolvedImportsMap, unresolvedOptionalImportsMap, - swiftModuleDependents, allDiscoveredClangModules, - failedToResolveImports, resolvedClangDependenciesMap); + allDiscoveredClangModules, failedToResolveImports, + resolvedClangDependenciesMap); // Re-query some failed-to-resolve Clang imports from cache // in chance they were brought in as transitive dependencies. @@ -1545,7 +1584,7 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule( bridgingHeaderCommandLine); moduleDependencyInfo.setHeaderSourceFiles(headerFileInputs); // Update the set of visible Clang modules - moduleDependencyInfo.addVisibleClangModules( + moduleDependencyInfo.setHeaderVisibleClangModules( headerScanResult->VisibleModules); // Update the dependency in the cache DependencyCache.updateDependency(moduleID, moduleDependencyInfo); @@ -1562,7 +1601,7 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( PrettyStackTraceStringAction trace( "Resolving Swift Overlay dependencies of module", moduleID.ModuleName); auto visibleClangDependencies = - DependencyCache.getVisibleClangModules(moduleID); + DependencyCache.getAllVisibleClangModules(moduleID); llvm::StringMap swiftOverlayLookupResult; std::mutex lookupResultLock; @@ -1968,8 +2007,8 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining( mainModuleDeps.setChainedBridgingHeaderBuffer( sourceBuffer->getBufferIdentifier(), sourceBuffer->getBuffer()); // Update the set of visible Clang modules - mainModuleDeps.addVisibleClangModules(headerScanResult->VisibleModules); - + mainModuleDeps.setHeaderVisibleClangModules( + headerScanResult->VisibleModules); // Update the dependency in the cache DependencyCache.updateDependency(rootModuleID, mainModuleDeps); return llvm::Error::success(); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index db21e0b3c617d..dee1b368c2f2d 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -1602,7 +1602,6 @@ void swift::dependencies::incremental::validateInterModuleDependenciesCache( emitRemarks, visited, modulesRequiringRescan); for (const auto &outOfDateModID : modulesRequiringRescan) cache.removeDependency(outOfDateModID); - // Regardless of invalidation, always re-scan main module. cache.removeDependency(rootModuleID); } diff --git a/test/ScanDependencies/basic_query_metrics.swift b/test/ScanDependencies/basic_query_metrics.swift index 4b47ff4dc7cb4..a83891cf5b816 100644 --- a/test/ScanDependencies/basic_query_metrics.swift +++ b/test/ScanDependencies/basic_query_metrics.swift @@ -9,7 +9,7 @@ // Ensure that despite being a common dependency to multiple Swift modules, only 1 query is performed to find 'C' // CHECK: remark: Number of Swift module queries: '6' // CHECK: remark: Number of named Clang module queries: '1' -// CHEKC: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '1' +// CHECK: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '1' // CHECK: remark: Number of recorded Swift module dependencies: '2' // CHECK: remark: Number of recorded Clang module dependencies: '1' diff --git a/test/ScanDependencies/error_source_locations.swift b/test/ScanDependencies/error_source_locations.swift index 2a57a2e51bba6..222b3384aeb5f 100644 --- a/test/ScanDependencies/error_source_locations.swift +++ b/test/ScanDependencies/error_source_locations.swift @@ -6,16 +6,6 @@ import P import FooBar - -// CHECK: {{.*}}{{/|\\}}error_source_locations.swift:7:8: error: unable to resolve module dependency: 'FooBar' -// CHECK-NEXT: 5 | -// CHECK-NEXT: 6 | import P -// CHECK-NEXT: 7 | import FooBar -// CHECK-NEXT: | |- error: unable to resolve module dependency: 'FooBar' -// CHECK-NEXT: | `- note: a dependency of main module 'deps' -// CHECK-NEXT: 8 | -// CHECK-NEXT: 9 | - // CHECK: {{.*}}{{/|\\}}Z.swiftinterface:3:8: error: unable to resolve module dependency: 'missing_module' // CHECK-NEXT: 1 | // swift-interface-format-version: 1.0 // CHECK-NEXT: 2 | // swift-module-flags: -module-name Z @@ -26,3 +16,12 @@ import FooBar // CHECK-NEXT: | |- note: a dependency of Swift module 'P': '{{.*}}{{/|\\}}P.swiftinterface' // CHECK-NEXT: | `- note: a dependency of main module 'deps' // CHECK-NEXT: 4 | public func funcZ() { } + +// CHECK: {{.*}}{{/|\\}}error_source_locations.swift:7:8: error: unable to resolve module dependency: 'FooBar' +// CHECK-NEXT: 5 | +// CHECK-NEXT: 6 | import P +// CHECK-NEXT: 7 | import FooBar +// CHECK-NEXT: | |- error: unable to resolve module dependency: 'FooBar' +// CHECK-NEXT: | `- note: a dependency of main module 'deps' +// CHECK-NEXT: 8 | +// CHECK-NEXT: 9 | diff --git a/test/ScanDependencies/module_deps_cache_reuse.swift b/test/ScanDependencies/module_deps_cache_reuse.swift index 62f808df56f35..ff1b3a9fedec3 100644 --- a/test/ScanDependencies/module_deps_cache_reuse.swift +++ b/test/ScanDependencies/module_deps_cache_reuse.swift @@ -28,9 +28,7 @@ import SubE // CHECK-REMARK-SAVE: remark: Incremental module scan: Serializing module scanning dependency cache to: // CHECK-REMARK-LOAD: remark: Incremental module scan: Re-using serialized module scanning dependency cache from: -// FIXME: Today, we do not serialize dependencies of the main source module which results in a lookup for 'C' even though -// it is fully redundant. -// CHECK-REMARK-LOAD: remark: Number of named Clang module queries: '1' +// CHECK-REMARK-LOAD: remark: Number of named Clang module queries: '0' // CHECK-REMARK-LOAD: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '0' // CHECK-REMARK-LOAD: remark: Number of recorded Swift module dependencies: '8' // CHECK-REMARK-LOAD: remark: Number of recorded Clang module dependencies: '7' From eff05cf285ea262a35e2d1c1062feef02adb4568 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Thu, 18 Dec 2025 10:40:21 -0800 Subject: [PATCH 2/2] [Dependency Scanning] Serialize visible Clang module sets for each by-name queried Clang module Re-use them on successive incremental scanning actions --- .../SerializedModuleDependencyCacheFormat.h | 13 ++++- .../ModuleDependencyCacheSerialization.cpp | 47 ++++++++++++++++ .../duplicate_clang_queries_on_overlay.swift | 53 +++++++++++++++++++ 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test/ScanDependencies/duplicate_clang_queries_on_overlay.swift diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index e1a324ef1b5a8..b81bd33bb9074 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -41,7 +41,7 @@ using llvm::BCVBR; const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'}; const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 10; /// Increment this on every change. -const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 4; +const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 5; /// Various identifiers in this format will rely on having their strings mapped /// using this ID. @@ -81,6 +81,7 @@ using FileIDArrayIDField = IdentifierIDField; using ContextHashIDField = IdentifierIDField; using ModuleCacheKeyIDField = IdentifierIDField; using ImportArrayIDField = IdentifierIDField; +using VisibleModulesArrayIDField = IdentifierIDField; using LinkLibrariesArrayIDField = IdentifierIDField; using MacroDependenciesArrayIDField = IdentifierIDField; using SearchPathArrayIDField = IdentifierIDField; @@ -109,6 +110,7 @@ enum { MACRO_DEPENDENCY_ARRAY_NODE, SEARCH_PATH_NODE, SEARCH_PATH_ARRAY_NODE, + VISIBLE_MODULES_NODE, IMPORT_STATEMENT_NODE, IMPORT_STATEMENT_ARRAY_NODE, OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE, @@ -187,6 +189,14 @@ using SearchPathLayout = using SearchPathArrayLayout = BCRecordLayout; +// A record capturing information about all Clang modules visible +// from a given named Clang module dependency query +using VisibleModulesLayout = + BCRecordLayout; + // A record capturing information about a given 'import' statement // captured in a dependency node, including its source location. using ImportStatementLayout = @@ -199,6 +209,7 @@ using ImportStatementLayout = IsExportedImport, // isExported AccessLevelField // accessLevel >; + using ImportStatementArrayLayout = BCRecordLayout; using OptionalImportStatementArrayLayout = diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index c3ed6f2fb8248..86ee58ee819ea 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -411,6 +411,23 @@ bool ModuleDependenciesCacheDeserializer::readGraph( break; } + case VISIBLE_MODULES_NODE: { + uint64_t identifierID; + uint64_t valueArrayID; + VisibleModulesLayout::readRecord(Scratch, identifierID, valueArrayID); + auto moduleName = getIdentifier(identifierID); + if (!moduleName) + llvm::report_fatal_error( + "Bad visible modules info: no module name"); + auto values = getStringArray(valueArrayID); + if (!values) + llvm::report_fatal_error( + "Bad visible modules info: modules"); + + cache.clangModulesVisibleFromNamedLookup[moduleName.value()] = values.value(); + break; + } + case IMPORT_STATEMENT_NODE: { unsigned importIdentifierID, bufferIdentifierID; unsigned lineNumber, columnNumber; @@ -1056,6 +1073,7 @@ enum ModuleIdentifierArrayKind : uint8_t { BridgingHeaderBuildCommandLine, NonPathCommandLine, FileDependencies, + VisibleClangModulesFromLookup, LastArrayKind }; @@ -1166,6 +1184,7 @@ class ModuleDependenciesCacheSerializer { unsigned writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo); void writeSearchPathsArray(unsigned startIndex, unsigned count); + void writeVisibleClangModuleInfo(const ModuleDependenciesCache &cache); void writeImportStatementInfos(const ModuleDependenciesCache &cache); unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo, bool optional); @@ -1435,6 +1454,21 @@ void ModuleDependenciesCacheSerializer::writeSearchPathsArray(unsigned startInde Out, ScratchRecord, AbbrCodes[SearchPathArrayLayout::Code], vec); } +void ModuleDependenciesCacheSerializer::writeVisibleClangModuleInfo( + const ModuleDependenciesCache &cache) { + using namespace graph_block; + for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) { + auto moduleID = + ModuleDependencyID{entry.getKey().str(), ModuleDependencyKind::Clang}; + VisibleModulesLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[VisibleModulesLayout::Code], + getIdentifier(moduleID.ModuleName), + getIdentifierArrayID( + moduleID, + ModuleIdentifierArrayKind::VisibleClangModulesFromLookup)); + } +} + void ModuleDependenciesCacheSerializer::writeImportStatementInfos( const ModuleDependenciesCache &cache) { unsigned lastImportInfoIndex = 0; @@ -1798,6 +1832,15 @@ unsigned ModuleDependenciesCacheSerializer::getOptionalImportStatementsArrayID( void ModuleDependenciesCacheSerializer::collectStringsAndArrays( const ModuleDependenciesCache &cache) { addIdentifier(cache.scannerContextHash); + + for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) { + auto moduleName = entry.getKey().str(); + addIdentifier(moduleName); + addStringArray({moduleName, ModuleDependencyKind::Clang}, + ModuleIdentifierArrayKind::VisibleClangModulesFromLookup, + entry.second); + } + for (auto kind = ModuleDependencyKind::FirstKind; kind != ModuleDependencyKind::LastKind; ++kind) { auto modMap = cache.getDependenciesMap(kind); @@ -1973,6 +2016,7 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); + registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); @@ -1998,6 +2042,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( // Write the arrays writeArraysOfIdentifiers(); + // Write the cached sets of visible modules + writeVisibleClangModuleInfo(cache); + // Write all the import statement info writeImportStatementInfos(cache); diff --git a/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift b/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift new file mode 100644 index 0000000000000..bf39ac0f59f7a --- /dev/null +++ b/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift @@ -0,0 +1,53 @@ +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/module-cache) +// RUN: %empty-directory(%t/inputs) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -module-cache-path %t/module-cache -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib %t/test.swift -o %t/deps.json -I %t/inputs -Rdependency-scan -no-parallel-scan &> %t/remarks.txt +// RUN: cat %t/remarks.txt | %FileCheck %s + +// Ensure that although the Swift overlay dependency 'A' shares a dependency on 'B' and 'C' +// it does not incur additional namedqueries for them. +// +// CHECK: remark: Number of Swift module queries: '9' +// CHECK: remark: Number of named Clang module queries: '2' +// CHECK: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '2' +// CHECK: remark: Number of recorded Swift module dependencies: '1' +// CHECK: remark: Number of recorded Clang module dependencies: '3' + +//--- test.swift +import B +import C +public func test() {} + +//--- inputs/A.swiftinterface +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -user-module-version 1.0 +import B +import C +public func a() { } + +//--- inputs/A.h +#include "A.h" +void b(void); + +//--- inputs/B.h +#include "A.h" +void b(void); + +//--- inputs/C.h +void c(void); + +//--- inputs/module.modulemap +module A { + header "A.h" + export * +} +module B { + header "B.h" + export * +} +module C { + header "C.h" + export * +}