diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index f70410465355..ed1a0443d31e 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -277,10 +277,9 @@ class ModuleDependencyInfoStorageBase { /// The macro dependencies. std::map macroDependencies; - /// A list of Clang modules that are visible to this Swift module. This - /// includes both direct Clang modules as well as transitive Clang - /// module dependencies when they are exported - llvm::StringSet<> visibleClangModules; + /// A list of Clang modules that are visible to this Swift module + /// as re-exported modular includes of its bridging header. + std::vector bridgingHeaderVisibleClangModules; /// ModuleDependencyInfo is finalized (with all transitive dependencies /// and inputs). @@ -893,14 +892,12 @@ class ModuleDependencyInfo { llvm_unreachable("Unexpected module dependency kind"); } - llvm::StringSet<> &getVisibleClangModules() const { - return storage->visibleClangModules; + ArrayRef getHeaderVisibleClangModules() const { + return storage->bridgingHeaderVisibleClangModules; } - - void - addVisibleClangModules(const std::vector &moduleNames) const { - storage->visibleClangModules.insert(moduleNames.begin(), - moduleNames.end()); + void setHeaderVisibleClangModules( + const std::vector &moduleNames) const { + storage->bridgingHeaderVisibleClangModules = moduleNames; } /// Whether explicit input paths of all the module dependencies @@ -1073,6 +1070,9 @@ class ModuleDependenciesCache { private: /// Discovered dependencies ModuleDependenciesKindMap ModuleDependenciesMap; + /// A map from Clang module name to all visible modules to a client + /// of a by-name import of this Clang module + llvm::StringMap> clangModulesVisibleFromNamedLookup; /// Set containing all of the Clang modules that have already been seen. llvm::DenseSet alreadySeenClangModules; /// Name of the module under scan @@ -1111,6 +1111,8 @@ class ModuleDependenciesCache { bool hasDependency(StringRef moduleName) const; /// Whether we have cached dependency information for the given Swift module. bool hasSwiftDependency(StringRef moduleName) const; + /// Whether we have cached dependency information for the given Clang module. + bool hasClangDependency(StringRef moduleName) const; /// Report the number of recorded Clang dependencies int numberOfClangDependencies() const; /// Report the number of recorded Swift dependencies @@ -1152,9 +1154,20 @@ class ModuleDependenciesCache { /// Query all cross-import overlay dependencies llvm::ArrayRef getCrossImportOverlayDependencies(const ModuleDependencyID &moduleID) const; + /// Query all visible Clang modules for a given Swift dependency - llvm::StringSet<>& - getVisibleClangModules(ModuleDependencyID moduleID) const; + llvm::StringSet<> + getAllVisibleClangModules(ModuleDependencyID moduleID) const; + /// Query all Clang modules visible via a by-name lookup of given + /// Clang dependency + llvm::ArrayRef + getVisibleClangModulesFromLookup(StringRef moduleName) const; + bool hasVisibleClangModulesFromLookup(StringRef moduleName) const; + /// Query all Clang modules visible from a given Swift module's + /// bridging header + llvm::ArrayRef + getVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const; + bool hasVisibleClangModulesViaHeader(ModuleDependencyID moduleID) const; /// Look for module dependencies for a module with the given ID /// @@ -1230,8 +1243,8 @@ class ModuleDependenciesCache { const ModuleDependencyIDCollectionView dependencyIDs); /// Add to this module's set of visible Clang modules void - addVisibleClangModules(ModuleDependencyID moduleID, - const std::vector &moduleNames); + setVisibleClangModulesFromLookup(ModuleDependencyID clangModuleID, + const std::vector &moduleNames); StringRef getMainModuleName() const { return mainScanModuleName; } diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h index 3f803bd6817c..1a1634172e4a 100644 --- a/include/swift/DependencyScan/ModuleDependencyScanner.h +++ b/include/swift/DependencyScan/ModuleDependencyScanner.h @@ -37,6 +37,14 @@ using ImportStatementInfoMap = std::unordered_map>; +/// A map from a module ID to a collection of module IDs. +using ModuleIDToModuleIDSetVectorMap = + std::unordered_map; + +using ModuleIDImportInfoPair = + std::pair; + struct ScannerMetrics { /// Number of performed queries for a Swift dependency with a given name std::atomic SwiftModuleQueries; @@ -306,7 +314,7 @@ class ModuleDependencyScanner { void resolveSwiftModuleDependencies( const ModuleDependencyID &rootModuleID, ModuleDependencyIDSetVector &discoveredSwiftModules); - void resolveAllClangModuleDependencies( + void resolveClangModuleDependencies( ArrayRef swiftModules, ModuleDependencyIDSetVector &discoveredClangModules); void resolveHeaderDependencies( @@ -390,25 +398,19 @@ class ModuleDependencyScanner { /// in \c failedToResolveImports. /// 4. Update the set of resolved Clang dependencies for each Swift /// module dependency in \c resolvedClangDependenciesMap. - void cacheComputedClangModuleLookupResults( + void processBatchClangModuleQueryResult( const BatchClangModuleLookupResult &lookupResult, const ImportStatementInfoMap &unresolvedImportsMap, const ImportStatementInfoMap &unresolvedOptionalImportsMap, - ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules, - std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap); + std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap); /// Re-query some failed-to-resolve Clang imports from cache /// in chance they were brought in as transitive dependencies. void reQueryMissedModulesFromCache( - const std::vector< - std::pair> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap); + const std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap); /// Assuming the \c `moduleImport` failed to resolve, /// iterate over all binary Swift module dependencies with serialized diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index e1a324ef1b5a..b81bd33bb907 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -41,7 +41,7 @@ using llvm::BCVBR; const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'}; const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 10; /// Increment this on every change. -const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 4; +const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 5; /// Various identifiers in this format will rely on having their strings mapped /// using this ID. @@ -81,6 +81,7 @@ using FileIDArrayIDField = IdentifierIDField; using ContextHashIDField = IdentifierIDField; using ModuleCacheKeyIDField = IdentifierIDField; using ImportArrayIDField = IdentifierIDField; +using VisibleModulesArrayIDField = IdentifierIDField; using LinkLibrariesArrayIDField = IdentifierIDField; using MacroDependenciesArrayIDField = IdentifierIDField; using SearchPathArrayIDField = IdentifierIDField; @@ -109,6 +110,7 @@ enum { MACRO_DEPENDENCY_ARRAY_NODE, SEARCH_PATH_NODE, SEARCH_PATH_ARRAY_NODE, + VISIBLE_MODULES_NODE, IMPORT_STATEMENT_NODE, IMPORT_STATEMENT_ARRAY_NODE, OPTIONAL_IMPORT_STATEMENT_ARRAY_NODE, @@ -187,6 +189,14 @@ using SearchPathLayout = using SearchPathArrayLayout = BCRecordLayout; +// A record capturing information about all Clang modules visible +// from a given named Clang module dependency query +using VisibleModulesLayout = + BCRecordLayout; + // A record capturing information about a given 'import' statement // captured in a dependency node, including its source location. using ImportStatementLayout = @@ -199,6 +209,7 @@ using ImportStatementLayout = IsExportedImport, // isExported AccessLevelField // accessLevel >; + using ImportStatementArrayLayout = BCRecordLayout; using OptionalImportStatementArrayLayout = diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index 0e843a8651bd..beb6040f437e 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -759,6 +759,9 @@ bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const { bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const { return findSwiftDependency(moduleName).has_value(); } +bool ModuleDependenciesCache::hasClangDependency(StringRef moduleName) const { + return findDependency(moduleName, ModuleDependencyKind::Clang).has_value(); +} int ModuleDependenciesCache::numberOfClangDependencies() const { return ModuleDependenciesMap.at(ModuleDependencyKind::Clang).size(); @@ -844,6 +847,16 @@ void ModuleDependenciesCache::recordClangDependency( } } +void ModuleDependenciesCache::setVisibleClangModulesFromLookup( + ModuleDependencyID moduleID, + const std::vector &visibleModules) { + ASSERT(moduleID.Kind == ModuleDependencyKind::Clang); + if (visibleModules.empty()) + return; + + clangModulesVisibleFromNamedLookup[moduleID.ModuleName] = visibleModules; +} + void ModuleDependenciesCache::updateDependency( ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) { auto &map = getDependenciesMap(moduleID.Kind); @@ -854,6 +867,10 @@ void ModuleDependenciesCache::updateDependency( void ModuleDependenciesCache::removeDependency(ModuleDependencyID moduleID) { auto &map = getDependenciesMap(moduleID.Kind); map.erase(moduleID.ModuleName); + // If we are removing a Clang module which was queried by-name + // in a prior scan, we must re-compute its set of visible modules. + if (moduleID.Kind == ModuleDependencyKind::Clang) + clangModulesVisibleFromNamedLookup.erase(moduleID.ModuleName); } void ModuleDependenciesCache::setImportedSwiftDependencies( @@ -936,22 +953,52 @@ ModuleDependencyIDCollectionView ModuleDependenciesCache::getAllDependencies( moduleInfo.getImportedClangDependencies()); } -void ModuleDependenciesCache::addVisibleClangModules( - ModuleDependencyID moduleID, const std::vector &moduleNames) { - if (moduleNames.empty()) - return; - auto dependencyInfo = findKnownDependency(moduleID); - auto updatedDependencyInfo = dependencyInfo; - updatedDependencyInfo.addVisibleClangModules(moduleNames); - updateDependency(moduleID, updatedDependencyInfo); +llvm::StringSet<> ModuleDependenciesCache::getAllVisibleClangModules( + ModuleDependencyID moduleID) const { + ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || + moduleID.Kind == ModuleDependencyKind::SwiftInterface || + moduleID.Kind == ModuleDependencyKind::SwiftBinary); + llvm::StringSet<> result; + if (hasVisibleClangModulesViaHeader(moduleID)) { + auto headerVisibleModules = getVisibleClangModulesViaHeader(moduleID); + result.insert(headerVisibleModules.begin(), headerVisibleModules.end()); + } + for (const auto &clangDepID : + findKnownDependency(moduleID).getImportedClangDependencies()) { + assert(hasVisibleClangModulesFromLookup(clangDepID.ModuleName)); + auto visibleModulesViaImport = + getVisibleClangModulesFromLookup(clangDepID.ModuleName); + result.insert(visibleModulesViaImport.begin(), + visibleModulesViaImport.end()); + } + return result; +} + +ArrayRef ModuleDependenciesCache::getVisibleClangModulesFromLookup( + StringRef moduleName) const { + ASSERT(hasVisibleClangModulesFromLookup(moduleName)); + return clangModulesVisibleFromNamedLookup.at(moduleName); +} + +bool ModuleDependenciesCache::hasVisibleClangModulesFromLookup( + StringRef moduleName) const { + return clangModulesVisibleFromNamedLookup.contains(moduleName); } -llvm::StringSet<> &ModuleDependenciesCache::getVisibleClangModules( +llvm::ArrayRef +ModuleDependenciesCache::getVisibleClangModulesViaHeader( + ModuleDependencyID moduleID) const { + ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || + moduleID.Kind == ModuleDependencyKind::SwiftInterface || + moduleID.Kind == ModuleDependencyKind::SwiftBinary); + return findKnownDependency(moduleID).getHeaderVisibleClangModules(); +} +bool ModuleDependenciesCache::hasVisibleClangModulesViaHeader( ModuleDependencyID moduleID) const { ASSERT(moduleID.Kind == ModuleDependencyKind::SwiftSource || moduleID.Kind == ModuleDependencyKind::SwiftInterface || moduleID.Kind == ModuleDependencyKind::SwiftBinary); - return findKnownDependency(moduleID).getVisibleClangModules(); + return !getVisibleClangModulesViaHeader(moduleID).empty(); } ModuleDependencyIDCollectionView diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index c3ed6f2fb824..86ee58ee819e 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -411,6 +411,23 @@ bool ModuleDependenciesCacheDeserializer::readGraph( break; } + case VISIBLE_MODULES_NODE: { + uint64_t identifierID; + uint64_t valueArrayID; + VisibleModulesLayout::readRecord(Scratch, identifierID, valueArrayID); + auto moduleName = getIdentifier(identifierID); + if (!moduleName) + llvm::report_fatal_error( + "Bad visible modules info: no module name"); + auto values = getStringArray(valueArrayID); + if (!values) + llvm::report_fatal_error( + "Bad visible modules info: modules"); + + cache.clangModulesVisibleFromNamedLookup[moduleName.value()] = values.value(); + break; + } + case IMPORT_STATEMENT_NODE: { unsigned importIdentifierID, bufferIdentifierID; unsigned lineNumber, columnNumber; @@ -1056,6 +1073,7 @@ enum ModuleIdentifierArrayKind : uint8_t { BridgingHeaderBuildCommandLine, NonPathCommandLine, FileDependencies, + VisibleClangModulesFromLookup, LastArrayKind }; @@ -1166,6 +1184,7 @@ class ModuleDependenciesCacheSerializer { unsigned writeSearchPaths(const SwiftBinaryModuleDependencyStorage &dependencyInfo); void writeSearchPathsArray(unsigned startIndex, unsigned count); + void writeVisibleClangModuleInfo(const ModuleDependenciesCache &cache); void writeImportStatementInfos(const ModuleDependenciesCache &cache); unsigned writeImportStatementInfos(const ModuleDependencyInfo &dependencyInfo, bool optional); @@ -1435,6 +1454,21 @@ void ModuleDependenciesCacheSerializer::writeSearchPathsArray(unsigned startInde Out, ScratchRecord, AbbrCodes[SearchPathArrayLayout::Code], vec); } +void ModuleDependenciesCacheSerializer::writeVisibleClangModuleInfo( + const ModuleDependenciesCache &cache) { + using namespace graph_block; + for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) { + auto moduleID = + ModuleDependencyID{entry.getKey().str(), ModuleDependencyKind::Clang}; + VisibleModulesLayout::emitRecord( + Out, ScratchRecord, AbbrCodes[VisibleModulesLayout::Code], + getIdentifier(moduleID.ModuleName), + getIdentifierArrayID( + moduleID, + ModuleIdentifierArrayKind::VisibleClangModulesFromLookup)); + } +} + void ModuleDependenciesCacheSerializer::writeImportStatementInfos( const ModuleDependenciesCache &cache) { unsigned lastImportInfoIndex = 0; @@ -1798,6 +1832,15 @@ unsigned ModuleDependenciesCacheSerializer::getOptionalImportStatementsArrayID( void ModuleDependenciesCacheSerializer::collectStringsAndArrays( const ModuleDependenciesCache &cache) { addIdentifier(cache.scannerContextHash); + + for (const auto &entry : cache.clangModulesVisibleFromNamedLookup) { + auto moduleName = entry.getKey().str(); + addIdentifier(moduleName); + addStringArray({moduleName, ModuleDependencyKind::Clang}, + ModuleIdentifierArrayKind::VisibleClangModulesFromLookup, + entry.second); + } + for (auto kind = ModuleDependencyKind::FirstKind; kind != ModuleDependencyKind::LastKind; ++kind) { auto modMap = cache.getDependenciesMap(kind); @@ -1973,6 +2016,7 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); + registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); registerRecordAbbr(); @@ -1998,6 +2042,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( // Write the arrays writeArraysOfIdentifiers(); + // Write the cached sets of visible modules + writeVisibleClangModuleInfo(cache); + // Write all the import statement info writeImportStatementInfos(cache); diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 6d492ae752e8..a4109a88b825 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -878,7 +878,7 @@ ModuleDependencyScanner::performDependencyScan(ModuleDependencyID rootModuleID) // If scanning for an individual Clang module, simply resolve its imports if (rootModuleID.Kind == ModuleDependencyKind::Clang) { ModuleDependencyIDSetVector discoveredClangModules; - resolveAllClangModuleDependencies({}, discoveredClangModules); + resolveClangModuleDependencies({}, discoveredClangModules); return discoveredClangModules.takeVector(); } @@ -932,8 +932,8 @@ ModuleDependencyScanner::resolveImportedModuleDependencies( // This operation is done by gathering all unresolved import // identifiers and querying them in-parallel to the Clang // dependency scanner. - resolveAllClangModuleDependencies(discoveredSwiftModules.getArrayRef(), - allModules); + resolveClangModuleDependencies(discoveredSwiftModules.getArrayRef(), + allModules); // For each discovered Swift module which was built with a // bridging header, scan the header for module dependencies. @@ -984,13 +984,22 @@ void ModuleDependencyScanner::resolveSwiftModuleDependencies( return; } -static void -gatherUnresolvedImports(ModuleDependenciesCache &cache, - ASTContext &scanASTContext, - ArrayRef swiftModuleDependents, - ModuleDependencyIDSetVector &allDiscoveredClangModules, - ImportStatementInfoMap &unresolvedImportsMap, - ImportStatementInfoMap &unresolvedOptionalImportsMap) { +static void findAllReachableClangModules(ModuleDependencyID moduleID, + const ModuleDependenciesCache &cache, + ModuleDependencyIDSetVector &reachableClangModules) { + if (!reachableClangModules.insert(moduleID)) + return; + for (const auto &depID : cache.getImportedClangDependencies(moduleID)) + findAllReachableClangModules(depID, cache, reachableClangModules); +} + +static void gatherUnresolvedImports( + ModuleDependenciesCache &cache, ASTContext &scanASTContext, + ArrayRef swiftModuleDependents, + ModuleDependencyIDSetVector &allDiscoveredClangModules, + ImportStatementInfoMap &unresolvedImportsMap, + ImportStatementInfoMap &unresolvedOptionalImportsMap, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { for (const auto &moduleID : swiftModuleDependents) { auto moduleDependencyInfo = cache.findKnownDependency(moduleID); auto unresolvedImports = @@ -1002,34 +1011,49 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, .emplace(moduleID, std::vector()) .first->second; - // If we have already resolved Clang dependencies for this module, + // If we have already fully resolved Clang dependencies for this module, // then we have the entire dependency sub-graph already computed for // it and ready to be added to 'allDiscoveredClangModules' without // additional scanning. if (!moduleDependencyInfo.getImportedClangDependencies().empty()) { auto directClangDeps = cache.getImportedClangDependencies(moduleID); ModuleDependencyIDSetVector reachableClangModules; - reachableClangModules.insert(directClangDeps.begin(), - directClangDeps.end()); - for (unsigned currentModuleIdx = 0; - currentModuleIdx < reachableClangModules.size(); - ++currentModuleIdx) { - auto moduleID = reachableClangModules[currentModuleIdx]; - auto dependencies = - cache.findKnownDependency(moduleID).getImportedClangDependencies(); - reachableClangModules.insert(dependencies.begin(), dependencies.end()); - } - allDiscoveredClangModules.insert(reachableClangModules.begin(), - reachableClangModules.end()); + for (const auto &depID : directClangDeps) + findAllReachableClangModules(depID, cache, reachableClangModules); + allDiscoveredClangModules.insert( + reachableClangModules.getArrayRef().begin(), + reachableClangModules.getArrayRef().end()); continue; } else { - // We need to query the Clang dependency scanner for this module's - // non-Swift imports llvm::StringSet<> resolvedImportIdentifiers; + // Mark all import identifiers which were resolved to Swift dependencies + // as resolved for (const auto &resolvedDep : moduleDependencyInfo.getImportedSwiftDependencies()) resolvedImportIdentifiers.insert(resolvedDep.ModuleName); + // Mark all import identifiers which can be fully resolved to a cached + // Clang dependency (including visible modules) as resolved + auto markCachedClangDependenciesResolved = + [&cache, &moduleID, &resolvedImportIdentifiers, + &resolvedClangDependenciesMap]( + ArrayRef imports) { + for (const auto &import : imports) { + auto &importIdentifier = import.importIdentifier; + if (!resolvedImportIdentifiers.contains(importIdentifier) && + cache.hasClangDependency(importIdentifier) && + cache.hasVisibleClangModulesFromLookup(importIdentifier)) { + resolvedImportIdentifiers.insert(importIdentifier); + resolvedClangDependenciesMap[moduleID].insert( + {importIdentifier, ModuleDependencyKind::Clang}); + } + } + }; + markCachedClangDependenciesResolved( + moduleDependencyInfo.getModuleImports()); + markCachedClangDependenciesResolved( + moduleDependencyInfo.getOptionalModuleImports()); + // When querying a *clang* module 'CxxStdlib' we must // instead expect a module called 'std'... auto addCanonicalClangModuleImport = @@ -1047,6 +1071,8 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, } }; + // We need to query the Clang dependency scanner for all of this module's + // unresolved imports for (const auto &depImport : moduleDependencyInfo.getModuleImports()) if (!resolvedImportIdentifiers.contains(depImport.importIdentifier)) addCanonicalClangModuleImport(depImport, *unresolvedImports); @@ -1059,10 +1085,8 @@ gatherUnresolvedImports(ModuleDependenciesCache &cache, } void ModuleDependencyScanner::reQueryMissedModulesFromCache( - const std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap) { + const std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { // It is possible that a specific import resolution failed because we are // attempting to resolve a module which can only be brought in via a // modulemap of a different Clang module dependency which is not otherwise @@ -1096,8 +1120,9 @@ void ModuleDependencyScanner::reQueryMissedModulesFromCache( if (optionalCachedModuleInfo) { resolvedClangDependenciesMap[unresolvedImport.first].insert( unresolvedModuleID); - DependencyCache.addVisibleClangModules( - unresolvedImport.first, {unresolvedImport.second.importIdentifier}); + DependencyCache.setVisibleClangModulesFromLookup( + unresolvedModuleID, + {unresolvedImport.second.importIdentifier}); ScanDiagnosticReporter.registerNamedClangDependency(); } else { // Failed to resolve module dependency. @@ -1156,61 +1181,76 @@ void ModuleDependencyScanner::performClangModuleLookup( ScanningThreadPool.wait(); } -void ModuleDependencyScanner::cacheComputedClangModuleLookupResults( +void ModuleDependencyScanner::processBatchClangModuleQueryResult( const BatchClangModuleLookupResult &lookupResult, const ImportStatementInfoMap &unresolvedImportsMap, const ImportStatementInfoMap &unresolvedOptionalImportsMap, - ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules, - std::vector> - &failedToResolveImports, - std::unordered_map - &resolvedClangDependenciesMap) { - for (const auto &moduleID : swiftModuleDependents) { - auto recordResolvedClangModuleImport = - [this, &lookupResult, &resolvedClangDependenciesMap, - &allDiscoveredClangModules, moduleID, &failedToResolveImports]( - const ScannerImportStatementInfo &moduleImport, - bool optionalImport) { - auto &moduleIdentifier = moduleImport.importIdentifier; - auto dependencyID = - ModuleDependencyID{moduleIdentifier, ModuleDependencyKind::Clang}; - - // Add visible Clang modules for this query to the depending - // Swift module - if (lookupResult.visibleModules.contains(moduleIdentifier)) - DependencyCache.addVisibleClangModules( - moduleID, lookupResult.visibleModules.at(moduleIdentifier)); - - // Add the resolved dependency ID + std::vector &failedToResolveImports, + ModuleIDToModuleIDSetVectorMap &resolvedClangDependenciesMap) { + + llvm::StringSet<> cachedNamedDependencies; + auto recordResolvedClangModuleImport = + [this, &lookupResult, &resolvedClangDependenciesMap, + &allDiscoveredClangModules, &failedToResolveImports, + &cachedNamedDependencies]( + const ModuleDependencyID &moduleID, + const ScannerImportStatementInfo &moduleImport, bool optionalImport) { + auto &moduleIdentifier = moduleImport.importIdentifier; + auto dependencyID = + ModuleDependencyID{moduleIdentifier, ModuleDependencyKind::Clang}; + + // A successful named query will have returned a set of visible modules + if (lookupResult.visibleModules.contains(moduleIdentifier)) { + // If this dependency has already been recorded in the cache, + // mark it as resolved for the current dependent and exit. + if (!cachedNamedDependencies.insert(moduleIdentifier).second) { + resolvedClangDependenciesMap[moduleID].insert(dependencyID); + return; + } + + ScanDiagnosticReporter.registerNamedClangDependency(); + DependencyCache.setVisibleClangModulesFromLookup( + dependencyID, lookupResult.visibleModules.at(moduleIdentifier)); + resolvedClangDependenciesMap[moduleID].insert(dependencyID); + + // If this module was not seen before as a transitive dependency + // of another lookup, record it into the cache if (lookupResult.discoveredDependencyInfos.contains( moduleIdentifier)) { - if (!DependencyCache.hasDependency(dependencyID)) - ScanDiagnosticReporter.registerNamedClangDependency(); - auto dependencyInfo = lookupResult.discoveredDependencyInfos.at( - moduleImport.importIdentifier); allDiscoveredClangModules.insert(dependencyID); DependencyCache.recordClangDependency( - dependencyInfo, ScanASTContext.Diags, [this](auto &clangDep) { + lookupResult.discoveredDependencyInfos.at( + moduleImport.importIdentifier), + ScanASTContext.Diags, [this](auto &clangDep) { return bridgeClangModuleDependency(clangDep); }); - resolvedClangDependenciesMap[moduleID].insert(dependencyID); - } else if (!optionalImport) { - // Otherwise, we failed to resolve this dependency. We will try - // again using the cache after all other imports have been - // resolved. If that fails too, a scanning failure will be - // diagnosed. - failedToResolveImports.push_back( - std::make_pair(moduleID, moduleImport)); + } else { + // If the query produced a set of visible modules but not + // a `ModuleDeps` info for the queried module itself, then + // it has to have been included in the set of already-seen + // module dependencies from a prior query. + assert(DependencyCache.hasDependency(dependencyID)); } - }; + } else if (!optionalImport) { + // Otherwise, we failed to resolve this dependency. We will try + // again using the cache after all other imports have been + // resolved. If that fails too, a scanning failure will be + // diagnosed. + failedToResolveImports.push_back( + std::make_pair(moduleID, moduleImport)); + } + }; - for (const auto &unresolvedImport : unresolvedImportsMap.at(moduleID)) - recordResolvedClangModuleImport(unresolvedImport, false); - for (const auto &unresolvedImport : - unresolvedOptionalImportsMap.at(moduleID)) - recordResolvedClangModuleImport(unresolvedImport, true); - } + for (const auto &moduleUnresolvedImports : unresolvedImportsMap) + for (const auto &unresolvedImport : moduleUnresolvedImports.second) + recordResolvedClangModuleImport(moduleUnresolvedImports.first, + unresolvedImport, false); + + for (const auto &moduleUnresolvedImports : unresolvedOptionalImportsMap) + for (const auto &unresolvedImport : moduleUnresolvedImports.second) + recordResolvedClangModuleImport(moduleUnresolvedImports.first, + unresolvedImport, true); // Use the computed scan results to record all transitive clang module // dependencies @@ -1227,33 +1267,32 @@ void ModuleDependencyScanner::cacheComputedClangModuleLookupResults( } } -void ModuleDependencyScanner::resolveAllClangModuleDependencies( +void ModuleDependencyScanner::resolveClangModuleDependencies( ArrayRef swiftModuleDependents, ModuleDependencyIDSetVector &allDiscoveredClangModules) { - // Gather all unresolved imports which must correspond to + // Gather all remaining unresolved imports which must correspond to // Clang modules (since no Swift module for them was found). ImportStatementInfoMap unresolvedImportsMap; ImportStatementInfoMap unresolvedOptionalImportsMap; - gatherUnresolvedImports(DependencyCache, ScanASTContext, swiftModuleDependents, - allDiscoveredClangModules, unresolvedImportsMap, - unresolvedOptionalImportsMap); + ModuleIDToModuleIDSetVectorMap resolvedClangDependenciesMap; + gatherUnresolvedImports(DependencyCache, ScanASTContext, + swiftModuleDependents, allDiscoveredClangModules, + unresolvedImportsMap, unresolvedOptionalImportsMap, + resolvedClangDependenciesMap); // Execute parallel lookup of all unresolved import // identifiers as Clang modules. BatchClangModuleLookupResult lookupResult; - performClangModuleLookup( - unresolvedImportsMap, unresolvedOptionalImportsMap, lookupResult); + performClangModuleLookup(unresolvedImportsMap, unresolvedOptionalImportsMap, + lookupResult); // Use the computed scan results to record directly-queried clang module // dependencies. - std::vector> - failedToResolveImports; - std::unordered_map - resolvedClangDependenciesMap; - cacheComputedClangModuleLookupResults( + std::vector failedToResolveImports; + processBatchClangModuleQueryResult( lookupResult, unresolvedImportsMap, unresolvedOptionalImportsMap, - swiftModuleDependents, allDiscoveredClangModules, - failedToResolveImports, resolvedClangDependenciesMap); + allDiscoveredClangModules, failedToResolveImports, + resolvedClangDependenciesMap); // Re-query some failed-to-resolve Clang imports from cache // in chance they were brought in as transitive dependencies. @@ -1545,7 +1584,7 @@ void ModuleDependencyScanner::resolveHeaderDependenciesForModule( bridgingHeaderCommandLine); moduleDependencyInfo.setHeaderSourceFiles(headerFileInputs); // Update the set of visible Clang modules - moduleDependencyInfo.addVisibleClangModules( + moduleDependencyInfo.setHeaderVisibleClangModules( headerScanResult->VisibleModules); // Update the dependency in the cache DependencyCache.updateDependency(moduleID, moduleDependencyInfo); @@ -1562,7 +1601,7 @@ void ModuleDependencyScanner::resolveSwiftOverlayDependenciesForModule( PrettyStackTraceStringAction trace( "Resolving Swift Overlay dependencies of module", moduleID.ModuleName); auto visibleClangDependencies = - DependencyCache.getVisibleClangModules(moduleID); + DependencyCache.getAllVisibleClangModules(moduleID); llvm::StringMap swiftOverlayLookupResult; std::mutex lookupResultLock; @@ -1968,8 +2007,8 @@ llvm::Error ModuleDependencyScanner::performBridgingHeaderChaining( mainModuleDeps.setChainedBridgingHeaderBuffer( sourceBuffer->getBufferIdentifier(), sourceBuffer->getBuffer()); // Update the set of visible Clang modules - mainModuleDeps.addVisibleClangModules(headerScanResult->VisibleModules); - + mainModuleDeps.setHeaderVisibleClangModules( + headerScanResult->VisibleModules); // Update the dependency in the cache DependencyCache.updateDependency(rootModuleID, mainModuleDeps); return llvm::Error::success(); diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index db21e0b3c617..dee1b368c2f2 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -1602,7 +1602,6 @@ void swift::dependencies::incremental::validateInterModuleDependenciesCache( emitRemarks, visited, modulesRequiringRescan); for (const auto &outOfDateModID : modulesRequiringRescan) cache.removeDependency(outOfDateModID); - // Regardless of invalidation, always re-scan main module. cache.removeDependency(rootModuleID); } diff --git a/test/ScanDependencies/basic_query_metrics.swift b/test/ScanDependencies/basic_query_metrics.swift index 4b47ff4dc7cb..a83891cf5b81 100644 --- a/test/ScanDependencies/basic_query_metrics.swift +++ b/test/ScanDependencies/basic_query_metrics.swift @@ -9,7 +9,7 @@ // Ensure that despite being a common dependency to multiple Swift modules, only 1 query is performed to find 'C' // CHECK: remark: Number of Swift module queries: '6' // CHECK: remark: Number of named Clang module queries: '1' -// CHEKC: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '1' +// CHECK: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '1' // CHECK: remark: Number of recorded Swift module dependencies: '2' // CHECK: remark: Number of recorded Clang module dependencies: '1' diff --git a/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift b/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift new file mode 100644 index 000000000000..bf39ac0f59f7 --- /dev/null +++ b/test/ScanDependencies/duplicate_clang_queries_on_overlay.swift @@ -0,0 +1,53 @@ +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/module-cache) +// RUN: %empty-directory(%t/inputs) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -module-cache-path %t/module-cache -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib %t/test.swift -o %t/deps.json -I %t/inputs -Rdependency-scan -no-parallel-scan &> %t/remarks.txt +// RUN: cat %t/remarks.txt | %FileCheck %s + +// Ensure that although the Swift overlay dependency 'A' shares a dependency on 'B' and 'C' +// it does not incur additional namedqueries for them. +// +// CHECK: remark: Number of Swift module queries: '9' +// CHECK: remark: Number of named Clang module queries: '2' +// CHECK: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '2' +// CHECK: remark: Number of recorded Swift module dependencies: '1' +// CHECK: remark: Number of recorded Clang module dependencies: '3' + +//--- test.swift +import B +import C +public func test() {} + +//--- inputs/A.swiftinterface +// swift-interface-format-version: 1.0 +// swift-module-flags: -module-name A -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -user-module-version 1.0 +import B +import C +public func a() { } + +//--- inputs/A.h +#include "A.h" +void b(void); + +//--- inputs/B.h +#include "A.h" +void b(void); + +//--- inputs/C.h +void c(void); + +//--- inputs/module.modulemap +module A { + header "A.h" + export * +} +module B { + header "B.h" + export * +} +module C { + header "C.h" + export * +} diff --git a/test/ScanDependencies/error_source_locations.swift b/test/ScanDependencies/error_source_locations.swift index 2a57a2e51bba..222b3384aeb5 100644 --- a/test/ScanDependencies/error_source_locations.swift +++ b/test/ScanDependencies/error_source_locations.swift @@ -6,16 +6,6 @@ import P import FooBar - -// CHECK: {{.*}}{{/|\\}}error_source_locations.swift:7:8: error: unable to resolve module dependency: 'FooBar' -// CHECK-NEXT: 5 | -// CHECK-NEXT: 6 | import P -// CHECK-NEXT: 7 | import FooBar -// CHECK-NEXT: | |- error: unable to resolve module dependency: 'FooBar' -// CHECK-NEXT: | `- note: a dependency of main module 'deps' -// CHECK-NEXT: 8 | -// CHECK-NEXT: 9 | - // CHECK: {{.*}}{{/|\\}}Z.swiftinterface:3:8: error: unable to resolve module dependency: 'missing_module' // CHECK-NEXT: 1 | // swift-interface-format-version: 1.0 // CHECK-NEXT: 2 | // swift-module-flags: -module-name Z @@ -26,3 +16,12 @@ import FooBar // CHECK-NEXT: | |- note: a dependency of Swift module 'P': '{{.*}}{{/|\\}}P.swiftinterface' // CHECK-NEXT: | `- note: a dependency of main module 'deps' // CHECK-NEXT: 4 | public func funcZ() { } + +// CHECK: {{.*}}{{/|\\}}error_source_locations.swift:7:8: error: unable to resolve module dependency: 'FooBar' +// CHECK-NEXT: 5 | +// CHECK-NEXT: 6 | import P +// CHECK-NEXT: 7 | import FooBar +// CHECK-NEXT: | |- error: unable to resolve module dependency: 'FooBar' +// CHECK-NEXT: | `- note: a dependency of main module 'deps' +// CHECK-NEXT: 8 | +// CHECK-NEXT: 9 | diff --git a/test/ScanDependencies/module_deps_cache_reuse.swift b/test/ScanDependencies/module_deps_cache_reuse.swift index 62f808df56f3..ff1b3a9fedec 100644 --- a/test/ScanDependencies/module_deps_cache_reuse.swift +++ b/test/ScanDependencies/module_deps_cache_reuse.swift @@ -28,9 +28,7 @@ import SubE // CHECK-REMARK-SAVE: remark: Incremental module scan: Serializing module scanning dependency cache to: // CHECK-REMARK-LOAD: remark: Incremental module scan: Re-using serialized module scanning dependency cache from: -// FIXME: Today, we do not serialize dependencies of the main source module which results in a lookup for 'C' even though -// it is fully redundant. -// CHECK-REMARK-LOAD: remark: Number of named Clang module queries: '1' +// CHECK-REMARK-LOAD: remark: Number of named Clang module queries: '0' // CHECK-REMARK-LOAD: remark: Number of recorded Clang module dependencies queried by-name from a Swift client: '0' // CHECK-REMARK-LOAD: remark: Number of recorded Swift module dependencies: '8' // CHECK-REMARK-LOAD: remark: Number of recorded Clang module dependencies: '7'