diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h index 60441f89a0a25..25464f4c29fbc 100644 --- a/lldb/include/lldb/Core/ModuleList.h +++ b/lldb/include/lldb/Core/ModuleList.h @@ -22,9 +22,12 @@ #include "lldb/lldb-types.h" #include "llvm/ADT/DenseSet.h" -#include "llvm/CAS/CASConfiguration.h" #include "llvm/Support/RWMutex.h" +// BEGIN CAS +#include "llvm/CAS/CASConfiguration.h" +// END CAS + #include #include #include @@ -33,6 +36,14 @@ #include #include +namespace llvm { +namespace cas { +class CASConfiguration; +class ObjectStore; +class ActionCache; +} // namespace cas +} // namespace llvm + namespace lldb_private { class ConstString; class FileSpecList; @@ -106,6 +117,7 @@ class ModuleListProperties : public Properties { // START CAS /// Get CASPath set via properties. FileSpec GetCASOnDiskPath() const; + bool SetCASOnDiskPath(const FileSpec &); /// Get CASPluginPath set via properties. FileSpec GetCASPluginPath() const; @@ -538,10 +550,14 @@ class ModuleList { // START CAS - /// Get CAS configuration using global module properties or from candidate - /// search path. - static std::optional - GetCASConfiguration(FileSpec CandidateConfigSearchPath); + struct CAS { + llvm::cas::CASConfiguration configuration; + std::shared_ptr object_store; + std::shared_ptr action_cache; + }; + /// Search for a CAS configuration file near this module. This + /// operation does a lot of file system stat calls. + static llvm::Expected GetOrCreateCAS(const lldb::ModuleSP &module_sp); /// Gets the shared module from CAS. It works the same as `GetSharedModule` /// but the lookup is done inside the CAS. @@ -550,11 +566,9 @@ class ModuleList { /// true if module is successfully loaded, false if module is not found /// in the CAS, error if there are any errors happened during the loading /// process. - static llvm::Expected GetSharedModuleFromCAS(ConstString module_name, - llvm::StringRef cas_id, - FileSpec cu_path, - ModuleSpec &module_spec, - lldb::ModuleSP &module_sp); + static llvm::Expected + GetSharedModuleFromCAS(llvm::StringRef cas_id, const lldb::ModuleSP &nearby, + ModuleSpec &module_spec, lldb::ModuleSP &module_sp); // END CAS static bool RemoveSharedModule(lldb::ModuleSP &module_sp); diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp index 699e2ac3da44e..22187bfce9296 100644 --- a/lldb/source/Core/ModuleList.cpp +++ b/lldb/source/Core/ModuleList.cpp @@ -17,6 +17,7 @@ #include "lldb/Interpreter/Property.h" #include "lldb/Symbol/ObjectFile.h" #include "lldb/Symbol/SymbolContext.h" +#include "lldb/Symbol/SymbolFile.h" #include "lldb/Symbol/TypeList.h" #include "lldb/Symbol/VariableList.h" #include "lldb/Utility/ArchSpec.h" @@ -27,7 +28,9 @@ #include "lldb/Utility/Log.h" #include "lldb/Utility/UUID.h" #include "lldb/lldb-defines.h" +#include "lldb/lldb-private-enumerations.h" #include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/Error.h" #include "llvm/Support/FileUtilities.h" #if defined(_WIN32) @@ -304,6 +307,11 @@ FileSpec ModuleListProperties::GetCASOnDiskPath() const { return GetPropertyAtIndexAs(idx, {}); } +bool ModuleListProperties::SetCASOnDiskPath(const FileSpec &path) { + const uint32_t idx = ePropertyCASOnDiskPath; + return SetPropertyAtIndex(idx, path); +} + FileSpec ModuleListProperties::GetCASPluginPath() const { const uint32_t idx = ePropertyCASPluginPath; return GetPropertyAtIndexAs(idx, {}); @@ -1278,6 +1286,17 @@ class SharedModuleList { continue; ModuleList to_remove = RemoveOrphansFromVector(vec); remove_count += to_remove.GetSize(); + // BEGIN CAS + to_remove.ForEach([&](auto &m) { + auto it = m_module_configs.find(m.get()); + if (it != m_module_configs.end()) + if (auto config = it->second) { + m_cas_cache.erase(config); + m_module_configs.erase(it); + } + return IterationAction::Continue; + }); + // END CAS m_list.Remove(to_remove); } // Break when fixed-point is reached. @@ -1292,6 +1311,21 @@ class SharedModuleList { /// filename, for fast module lookups by name. llvm::DenseMap> m_name_to_modules; + // BEGIN CAS +public: + /// Each module may have a CAS config associated with it. + /// Often many modules share the same CAS. + llvm::DenseMap m_module_configs; + + /// Each CAS config has a CAS associated with it. + llvm::DenseMap, + std::shared_ptr>> + m_cas_cache; + +private: + // END CAS + /// The use count of a module held only by m_list and m_name_to_modules. static constexpr long kUseCountSharedModuleListOrphaned = 2; }; @@ -1299,7 +1333,6 @@ class SharedModuleList { struct SharedModuleListInfo { SharedModuleList module_list; ModuleListProperties module_list_properties; - std::shared_ptr cas_object_store; std::mutex shared_lock; }; } @@ -1322,45 +1355,21 @@ static SharedModuleList &GetSharedModuleList() { return GetSharedModuleListInfo().module_list; } -std::optional -ModuleList::GetCASConfiguration(FileSpec CandidateConfigSearchPath) { +static llvm::cas::CASConfiguration +GetCASConfiguration(FileSpec CandidateConfigSearchPath) { // Config CAS from properties. - llvm::cas::CASConfiguration cas_config; - cas_config.CASPath = - ModuleList::GetGlobalModuleListProperties().GetCASOnDiskPath().GetPath(); - cas_config.PluginPath = - ModuleList::GetGlobalModuleListProperties().GetCASPluginPath().GetPath(); - cas_config.PluginOptions = - ModuleList::GetGlobalModuleListProperties().GetCASPluginOptions(); - - if (!cas_config.CASPath.empty()) - return cas_config; - + auto &props = ModuleList::GetGlobalModuleListProperties(); + auto path = props.GetCASOnDiskPath().GetPath(); + if (!path.empty()) { + return {props.GetCASOnDiskPath().GetPath(), + props.GetCASPluginPath().GetPath(), props.GetCASPluginOptions()}; + } auto search_config = llvm::cas::CASConfiguration::createFromSearchConfigFile( CandidateConfigSearchPath.GetPath()); if (search_config) return search_config->second; - return std::nullopt; -} - -static llvm::Expected> -GetOrCreateCASStorage(FileSpec CandidateConfigSearchPath) { - auto &shared_module_list = GetSharedModuleListInfo(); - if (shared_module_list.cas_object_store) - return shared_module_list.cas_object_store; - - auto config = ModuleList::GetCASConfiguration(CandidateConfigSearchPath); - if (!config) - return nullptr; - - auto cas = config->createDatabases(); - if (!cas) - return cas.takeError(); - - std::scoped_lock lock(shared_module_list.shared_lock); - shared_module_list.cas_object_store = std::move(cas->first); - return shared_module_list.cas_object_store; + return {}; } ModuleListProperties &ModuleList::GetGlobalModuleListProperties() { @@ -1634,19 +1643,18 @@ ModuleList::GetSharedModule(const ModuleSpec &module_spec, ModuleSP &module_sp, return error; } -static llvm::Expected loadModuleFromCAS(ConstString module_name, - llvm::StringRef cas_id, - FileSpec cu_path, - ModuleSpec &module_spec) { - auto maybe_cas = GetOrCreateCASStorage(cu_path); +static llvm::Expected loadModuleFromCASImpl(llvm::StringRef cas_id, + const lldb::ModuleSP &nearby, + ModuleSpec &module_spec) { + auto maybe_cas = ModuleList::GetOrCreateCAS(nearby); if (!maybe_cas) return maybe_cas.takeError(); - auto cas = std::move(*maybe_cas); + auto cas = std::move(maybe_cas->object_store); if (!cas) { LLDB_LOG(GetLog(LLDBLog::Modules), "skip loading module '{0}' from CAS: CAS is not available", - module_name); + cas_id); return false; } @@ -1665,20 +1673,142 @@ static llvm::Expected loadModuleFromCAS(ConstString module_name, std::make_shared(module_proxy->getMemoryBuffer()); // Swap out the module_spec with the one loaded via CAS. - ModuleSpec loaded(module_spec.GetFileSpec(), module_spec.GetUUID(), - std::move(file_buffer)); + FileSpec cas_spec; + cas_spec.SetDirectory(ConstString(maybe_cas->configuration.CASPath)); + cas_spec.SetFilename(ConstString(cas_id)); + ModuleSpec loaded(cas_spec, module_spec.GetUUID(), std::move(file_buffer)); loaded.GetArchitecture() = module_spec.GetArchitecture(); module_spec = loaded; - LLDB_LOG(GetLog(LLDBLog::Modules), "loading module '{0}' using CASID '{1}'", - module_name, cas_id); + LLDB_LOG(GetLog(LLDBLog::Modules), "loading module using CASID '{0}'", + cas_id); return true; } +/// Load the module referenced by \c cas_id from a CAS located +/// near \c nearby. +static llvm::Expected loadModuleFromCAS(llvm::StringRef cas_id, + const lldb::ModuleSP &nearby, + ModuleSpec &module_spec) { + static llvm::StringMap g_cache; + static std::recursive_mutex g_cache_lock; + std::scoped_lock lock(g_cache_lock); + auto cached = g_cache.find(cas_id); + if (cached != g_cache.end()) + return cached->second; + auto result = loadModuleFromCASImpl(cas_id, nearby, module_spec); + // Errors are only returned the first time. + g_cache.insert({cas_id, result ? *result : false}); + return result; +} + +static llvm::cas::CASConfiguration +FindCASConfiguration(const ModuleSP &module_sp) { + auto get_dir = [](FileSpec path) { + path.ClearFilename(); + return path; + }; + + // Look near the binary / dSYM. + std::set unique_paths; + llvm::cas::CASConfiguration cas_config = + GetCASConfiguration(module_sp->GetFileSpec()); + + if (!cas_config) { + // Look near the object files. + if (SymbolFile *sf = module_sp->GetSymbolFile()) { + sf->GetDebugInfoModules().ForEach([&](const ModuleSP &m) { + if (m) + unique_paths.insert(get_dir(m->GetFileSpec())); + return IterationAction::Continue; + }); + for (auto &path : unique_paths) + if ((cas_config = GetCASConfiguration(path))) + break; + } + } + // TODO: Remove this block once the build system consistently + // generates .cas-config files. + if (!cas_config) { + unique_paths.insert(get_dir(module_sp->GetFileSpec())); + for (auto &path : unique_paths) { + llvm::StringRef parent = path.GetDirectory().GetStringRef(); + while (!parent.empty() && + llvm::sys::path::filename(parent) != "DerivedData") + parent = llvm::sys::path::parent_path(parent); + if (parent.empty()) + continue; + llvm::SmallString<256> cas_path(parent); + llvm::sys::path::append(cas_path, "CompilationCache.noindex", "builtin"); + FileSpec fs = FileSpec(cas_path); + ModuleList::GetGlobalModuleListProperties().SetCASOnDiskPath(fs); + cas_config = GetCASConfiguration(fs); + if (cas_config) + break; + } + } + return cas_config; +} + +llvm::Expected +ModuleList::GetOrCreateCAS(const ModuleSP &module_sp) { + if (!module_sp) + return llvm::createStringError("no lldb::Module available"); + + // Look in cache first. + auto &shared_module_list = GetSharedModuleListInfo(); + std::scoped_lock lock(shared_module_list.shared_lock); + auto &module_configs = shared_module_list.module_list.m_module_configs; + auto &cas_cache = shared_module_list.module_list.m_cas_cache; + + llvm::cas::CASConfiguration cas_config; + { + auto cached_config = module_configs.find(module_sp.get()); + if (cached_config != module_configs.end()) { + cas_config = cached_config->second; + if (!cas_config) + return llvm::createStringError("no CAS available (cached)"); + } + } + + if (!cas_config) { + cas_config = FindCASConfiguration(module_sp); + // Cache the config or lack thereof. + module_configs.insert({module_sp.get(), cas_config}); + } + + if (!cas_config) + return llvm::createStringError("no CAS available"); + + // Look in the cache. + { + auto cached = cas_cache.find(cas_config); + if (cached != cas_cache.end()) { + if (!cached->second.first) + return llvm::createStringError( + "CAS config created, but CAS not available (cached)"); + return ModuleList::CAS{cas_config, cached->second.first, + cached->second.second}; + } + } + + // Create the CAS. + auto cas = cas_config.createDatabases(); + if (!cas) { + cas_cache.insert({cas_config, {}}); + return cas.takeError(); + } + + LLDB_LOG(GetLog(LLDBLog::Modules | LLDBLog::Symbols), + "Initialized CAS at {0}", cas_config.CASPath); + cas_cache.insert({cas_config, {cas->first, cas->second}}); + return ModuleList::CAS{cas_config, cas->first, cas->second}; +} + llvm::Expected ModuleList::GetSharedModuleFromCAS( - ConstString module_name, llvm::StringRef cas_id, FileSpec cu_path, + llvm::StringRef cas_id, const lldb::ModuleSP &nearby, ModuleSpec &module_spec, lldb::ModuleSP &module_sp) { - auto loaded = loadModuleFromCAS(module_name, cas_id, cu_path, module_spec); + auto loaded = loadModuleFromCAS(cas_id, nearby, module_spec); if (!loaded) return loaded.takeError(); @@ -1688,8 +1818,17 @@ llvm::Expected ModuleList::GetSharedModuleFromCAS( auto status = GetSharedModule(module_spec, module_sp, nullptr, nullptr, nullptr, /*always_create=*/true); - if (status.Success()) + if (status.Success()) { + if (module_sp) { + // Enter the new module into the config cache. + auto &shared_module_list = GetSharedModuleListInfo(); + std::scoped_lock lock(shared_module_list.shared_lock); + auto &module_configs = shared_module_list.module_list.m_module_configs; + auto config = module_configs.lookup(nearby.get()); + module_configs.insert({module_sp.get(), config}); + } return true; + } return status.takeError(); } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index b53d0e5fd9381..891bf1b6bde05 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -2007,18 +2007,21 @@ void SymbolFileDWARF::UpdateExternalModuleListIfNeeded() { dwo_module_spec.GetArchitecture() = m_objfile_sp->GetModule()->GetArchitecture(); - // Try load from CAS, if loaded, continue to next one. - auto loaded = ModuleList::GetSharedModuleFromCAS( - const_name, dwo_path, GetObjectFile()->GetFileSpec(), dwo_module_spec, - module_sp); - if (!loaded) - GetObjectFile()->GetModule()->ReportWarning( - "Failed to load module '{0}' from CAS: {1}", const_name, - toString(loaded.takeError())); - - // succeed, loaded next module. - if (*loaded) - continue; + // BEGIN CAS + if (!FileSystem::Instance().Exists(dwo_path)) { + LLDB_LOG(GetLog(LLDBLog::Symbols | LLDBLog::Modules), + "Loading module '{0}' from CAS at {1}...", const_name, dwo_path); + auto loaded = ModuleList::GetSharedModuleFromCAS( + dwo_path, GetObjectFile()->GetModule(), dwo_module_spec, module_sp); + if (!loaded) + GetObjectFile()->GetModule()->ReportWarning( + "Failed to load module '{0}' from CAS: {1}", const_name, + toString(loaded.takeError())); + else if (*loaded) + // Succeed, load next module. + continue; + } + // END CAS dwo_module_spec.GetFileSpec().SetFile(dwo_path, FileSpec::Style::native); if (dwo_module_spec.GetFileSpec().IsRelative()) { diff --git a/lldb/source/Plugins/TypeSystem/Swift/CMakeLists.txt b/lldb/source/Plugins/TypeSystem/Swift/CMakeLists.txt index c7829de818254..e12ffd33b9eda 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/CMakeLists.txt +++ b/lldb/source/Plugins/TypeSystem/Swift/CMakeLists.txt @@ -1,4 +1,5 @@ add_lldb_library(lldbPluginTypeSystemSwift PLUGIN + LLDBExplicitModuleLoader.cpp SwiftDWARFImporterForClangTypes.cpp TypeSystemSwift.cpp TypeSystemSwiftTypeRef.cpp diff --git a/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.cpp b/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.cpp new file mode 100644 index 0000000000000..cc6def9e6773e --- /dev/null +++ b/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.cpp @@ -0,0 +1,134 @@ +//===--LLDBExplicitModuleLoader.cpp --------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===---------------------------------------------------------------------===// + +#include "LLDBExplicitModuleLoader.h" +#include "swift/Serialization/SerializedModuleLoader.h" +#include +namespace lldb_private { + +LLDBExplicitSwiftModuleLoader::LLDBExplicitSwiftModuleLoader( + swift::ASTContext &ctx, llvm::cas::ObjectStore *CAS, + swift::DependencyTracker *tracker, swift::ModuleLoadingMode loadMode, + bool IgnoreSwiftSourceInfoFile, + std::unique_ptr casml, + std::unique_ptr esml) + : swift::SerializedModuleLoaderBase(ctx, tracker, loadMode, + IgnoreSwiftSourceInfoFile), + m_cas(CAS), m_casml(std::move(casml)), m_esml(std::move(esml)) {} + +std::unique_ptr +LLDBExplicitSwiftModuleLoader::create( + swift::ASTContext &ctx, llvm::cas::ObjectStore *CAS, + llvm::cas::ActionCache *cache, swift::DependencyTracker *tracker, + swift::ModuleLoadingMode loadMode, llvm::StringRef ExplicitSwiftModuleMap, + const llvm::StringMap &ExplicitSwiftModuleInputs, + bool IgnoreSwiftSourceInfoFile) { + auto esml = swift::ExplicitSwiftModuleLoader::create( + ctx, tracker, loadMode, ExplicitSwiftModuleMap, ExplicitSwiftModuleInputs, + IgnoreSwiftSourceInfoFile); + std::unique_ptr casml; + if (CAS && cache) { + casml = swift::ExplicitCASModuleLoader::create( + ctx, *CAS, *cache, tracker, loadMode, ExplicitSwiftModuleMap, + ExplicitSwiftModuleInputs, IgnoreSwiftSourceInfoFile); + } + return std::make_unique( + ctx, CAS, tracker, loadMode, IgnoreSwiftSourceInfoFile, std::move(casml), + std::move(esml)); +} + +void LLDBExplicitSwiftModuleLoader::collectVisibleTopLevelModuleNames( + llvm::SmallVectorImpl &names) const { + if (m_casml) + m_casml->collectVisibleTopLevelModuleNames(names); + m_esml->collectVisibleTopLevelModuleNames(names); +} + +std::error_code LLDBExplicitSwiftModuleLoader::findModuleFilesInDirectory( + swift::ImportPath::Element ModuleID, + const swift::SerializedModuleBaseName &BaseName, + llvm::SmallVectorImpl *ModuleInterfacePath, + llvm::SmallVectorImpl *ModuleInterfaceSourcePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer, + bool IsCanImportLookup, bool IsFramework, bool IsTestableDependencyLookup) { + // This is a protected member and probably also not useful. + return {}; +} +bool LLDBExplicitSwiftModuleLoader::canImportModule( + swift::ImportPath::Module named, swift::SourceLoc loc, + ModuleVersionInfo *versionInfo, bool isTestableImport) { + if (m_casml && + llvm::cast(m_casml.get()) + ->canImportModule(named, loc, versionInfo, isTestableImport)) + return true; + return llvm::cast(m_esml.get()) + ->canImportModule(named, loc, versionInfo, isTestableImport); +} + +swift::ModuleDecl * +LLDBExplicitSwiftModuleLoader::loadModule(swift::SourceLoc importLoc, + swift::ImportPath::Module path, + bool AllowMemoryCache) { + if (m_casml) + if (auto *decl = m_casml->loadModule(importLoc, path, AllowMemoryCache)) + return decl; + return m_esml->loadModule(importLoc, path, AllowMemoryCache); +} + +void LLDBExplicitSwiftModuleLoader::loadExtensions( + swift::NominalTypeDecl *nominal, unsigned previousGeneration) { + if (m_casml) + m_casml->loadExtensions(nominal, previousGeneration); + m_esml->loadExtensions(nominal, previousGeneration); +} + +void LLDBExplicitSwiftModuleLoader::loadObjCMethods( + swift::NominalTypeDecl *typeDecl, swift::ObjCSelector selector, + bool isInstanceMethod, unsigned previousGeneration, + llvm::TinyPtrVector &methods) { + if (m_casml) + m_casml->loadObjCMethods(typeDecl, selector, isInstanceMethod, + previousGeneration, methods); + m_esml->loadObjCMethods(typeDecl, selector, isInstanceMethod, + previousGeneration, methods); +} + +void LLDBExplicitSwiftModuleLoader::loadDerivativeFunctionConfigurations( + swift::AbstractFunctionDecl *originalAFD, unsigned previousGeneration, + llvm::SetVector &results) { + if (m_casml) + m_casml->loadDerivativeFunctionConfigurations(originalAFD, + previousGeneration, results); + m_esml->loadDerivativeFunctionConfigurations(originalAFD, previousGeneration, + results); +} + +void LLDBExplicitSwiftModuleLoader::verifyAllModules() { + if (m_casml) + m_casml->verifyAllModules(); + m_esml->verifyAllModules(); +} + +void LLDBExplicitSwiftModuleLoader::addExplicitModulePath(llvm::StringRef name, + std::string path) { + if (m_cas && m_casml) { + auto parsed_id = m_cas->parseID(path); + if (parsed_id) + return m_casml->addExplicitModulePath(name, path); + llvm::consumeError(parsed_id.takeError()); + } + m_esml->addExplicitModulePath(name, path); +} + +} // namespace lldb_private diff --git a/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.h b/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.h new file mode 100644 index 0000000000000..1ddc7b150fa23 --- /dev/null +++ b/lldb/source/Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.h @@ -0,0 +1,85 @@ +//===--LLDBExplicitModuleLoader.h -------------------------------*- C++-*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +#ifndef liblldb_LLDBExplicitModuleLoader_h_ +#define liblldb_LLDBExplicitModuleLoader_h_ + +#include +namespace swift { +class ExplicitSwiftModuleLoader; +class ExplicitCASModuleLoader; +} // namespace swift + +namespace lldb_private { +/// This Swift module loader implementation wraps a CAS module loader +/// and an ESML and forwards calls to addExplicitModulePath to both of +/// them. +class LLDBExplicitSwiftModuleLoader : public swift::SerializedModuleLoaderBase { +public: + LLDBExplicitSwiftModuleLoader( + swift::ASTContext &ctx, llvm::cas::ObjectStore *CAS, + swift::DependencyTracker *tracker, swift::ModuleLoadingMode LoadMode, + bool IgnoreSwiftSourceInfoFile, + std::unique_ptr casml, + std::unique_ptr esml); + + static std::unique_ptr + create(swift::ASTContext &ctx, llvm::cas::ObjectStore *CAS, + llvm::cas::ActionCache *cache, swift::DependencyTracker *tracker, + swift::ModuleLoadingMode loadMode, + llvm::StringRef ExplicitSwiftModuleMap, + const llvm::StringMap &ExplicitSwiftModuleInputs, + bool IgnoreSwiftSourceInfoFile); + + void collectVisibleTopLevelModuleNames( + llvm::SmallVectorImpl &names) const override; + std::error_code findModuleFilesInDirectory( + swift::ImportPath::Element ModuleID, + const swift::SerializedModuleBaseName &BaseName, + llvm::SmallVectorImpl *ModuleInterfacePath, + llvm::SmallVectorImpl *ModuleInterfaceSourcePath, + std::unique_ptr *ModuleBuffer, + std::unique_ptr *ModuleDocBuffer, + std::unique_ptr *ModuleSourceInfoBuffer, + bool IsCanImportLookup, bool IsFramework, + bool IsTestableDependencyLookup = false) override; + + bool canImportModule(swift::ImportPath::Module named, swift::SourceLoc loc, + ModuleVersionInfo *versionInfo, + bool isTestableImport = false) override; + + swift::ModuleDecl *loadModule(swift::SourceLoc importLoc, + swift::ImportPath::Module path, + bool AllowMemoryCache = true) override; + void loadExtensions(swift::NominalTypeDecl *nominal, + unsigned previousGeneration) override; + void loadObjCMethods( + swift::NominalTypeDecl *typeDecl, swift::ObjCSelector selector, + bool isInstanceMethod, unsigned previousGeneration, + llvm::TinyPtrVector &methods) override; + + void loadDerivativeFunctionConfigurations( + swift::AbstractFunctionDecl *originalAFD, unsigned previousGeneration, + llvm::SetVector &results) override; + + void verifyAllModules() override; + void addExplicitModulePath(llvm::StringRef name, std::string path) override; + +protected: + llvm::cas::ObjectStore *m_cas; + /// The CAS Swift module loader. + std::unique_ptr m_casml; + /// The explicit Swift module loader (ESML). + std::unique_ptr m_esml; +}; +} // namespace lldb_private +#endif diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp index 7de8d4bb4cc20..2bb46083fe3cd 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.cpp @@ -913,29 +913,27 @@ static std::string GetClangModulesCacheProperty() { return std::string(path); } -static void ConfigureCASStorage(SwiftASTContext *m_ast_context, - FileSpec CandidateConfigSearchPath) { - std::string m_description; - auto cas_config = ModuleList::GetCASConfiguration(CandidateConfigSearchPath); - if (!cas_config) { - HEALTH_LOG_PRINTF("no CAS available"); - return; - } - auto maybe_cas = cas_config->createDatabases(); - if (!maybe_cas) { - Debugger::ReportWarning("failed to create CAS: " + - toString(maybe_cas.takeError())); +/// This function implements various heuristics to find a CAS +/// configuration file. +static void ConfigureCASStorage(const std::string &m_description, + SwiftASTContext *m_ast_context, + const SymbolContext &sc) { + auto cas = ModuleList::GetOrCreateCAS(sc.module_sp); + if (!cas) { + HEALTH_LOG_PRINTF("Did not create CAS: %s", + toString(cas.takeError()).c_str()); return; } - m_ast_context->SetCASStorage(std::move(maybe_cas->first), - std::move(maybe_cas->second)); - m_ast_context->GetCASOptions().CASOpts.CASPath = cas_config->CASPath; - m_ast_context->GetCASOptions().CASOpts.PluginPath = cas_config->PluginPath; + m_ast_context->SetCASStorage(std::move(cas->object_store), + std::move(cas->action_cache)); + m_ast_context->GetCASOptions().CASOpts.CASPath = cas->configuration.CASPath; + m_ast_context->GetCASOptions().CASOpts.PluginPath = + cas->configuration.PluginPath; m_ast_context->GetCASOptions().CASOpts.PluginOptions = - cas_config->PluginOptions; + cas->configuration.PluginOptions; LOG_PRINTF(GetLog(LLDBLog::Types), "Setup CAS from module list properties with cas path: %s", - cas_config->CASPath.c_str()); + cas->configuration.CASPath.c_str()); } SwiftASTContext::ScopedDiagnostics::ScopedDiagnostics( @@ -1267,11 +1265,9 @@ static std::string GetPluginServerForSDK(llvm::StringRef sdk_path) { } namespace { - constexpr std::array g_known_eplicit_module_prefixes = - {"-fmodule-map-file=", - "-fmodule-file=", - "-fno-implicit-modules", - "-fno-implicit-module-maps"}; +constexpr std::array g_known_eplicit_module_prefixes = { + "-fmodule-map-file=", "-fmodule-file=", "-fmodule-file-cache-key=", + "-fno-implicit-modules", "-fno-implicit-module-maps"}; } /// Retrieve the serialized AST data blobs and initialize the compiler @@ -1862,8 +1858,10 @@ void SwiftASTContext::AddExtraClangArgs( m_has_explicit_modules |= llvm::any_of(importer_options.ExtraArgs, [](const std::string &s) { StringRef arg(s); - return arg.starts_with("-fno-implicit-module") || - arg.starts_with("-fmodule-file="); + for (const auto &option : g_known_eplicit_module_prefixes) + if (arg.starts_with(option)) + return true; + return false; }); ConfigureModuleValidation(importer_options.ExtraArgs); }); @@ -1906,8 +1904,7 @@ void SwiftASTContext::AddExtraClangArgs( bool SwiftASTContext::IsModuleAvailableInCAS(const std::string &key) { auto id = m_cas->parseID(key); if (!id) { - HEALTH_LOG_PRINTF("failed to parse CASID when loading module: %s", - toString(id.takeError()).c_str()); + llvm::consumeError(id.takeError()); return false; } auto lookup = m_action_cache->get(*id); @@ -1994,7 +1991,7 @@ void SwiftASTContext::AddExtraClangCC1Args( invocation.getFrontendOpts().ModuleCacheKeys, [&](const auto &entry) { bool exist = IsModuleAvailableInCAS(entry.second); if (!exist) - HEALTH_LOG_PRINTF("module '%s' cannot be load " + HEALTH_LOG_PRINTF("module '%s' cannot be loaded " "from CAS using key: %s, fallback to " "load from file system", entry.first.c_str(), entry.second.c_str()); @@ -2614,7 +2611,9 @@ SwiftASTContext::CreateInstance(lldb::LanguageType language, Module &module, } } - ConfigureCASStorage(swift_ast_sp.get(), module.GetFileSpec()); + SymbolContext sc; + module.CalculateSymbolContext(&sc); + ConfigureCASStorage(m_description, swift_ast_sp.get(), sc); // The serialized triple is the triple of the last binary // __swiftast section that was processed. Instead of relying on @@ -3097,7 +3096,7 @@ lldb::TypeSystemSP SwiftASTContext::CreateInstance( } } - ConfigureCASStorage(swift_ast_sp.get(), sc.module_sp->GetFileSpec()); + ConfigureCASStorage(m_description, swift_ast_sp.get(), sc); std::string resource_dir = HostInfo::GetSwiftResourceDir( triple, swift_ast_sp->GetPlatformSDKPath()); @@ -3838,30 +3837,19 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() { // The order here matters due to fallback behaviors: // - // 1. Create and install the memory buffer serialized module loader. - std::unique_ptr memory_buffer_loader_up( - swift::MemoryBufferSerializedModuleLoader::create( - *m_ast_context_up, m_dependency_tracker.get(), loading_mode, - /*IgnoreSwiftSourceInfo*/ false, /*BypassResilience*/ true)); - if (memory_buffer_loader_up) { - m_memory_buffer_module_loader = - static_cast( - memory_buffer_loader_up.get()); - m_ast_context_up->addModuleLoader(std::move(memory_buffer_loader_up)); - } - - // 2. Create the explicit swift module loader. + // 1. Create the explicit swift module loader. if (props.GetUseSwiftExplicitModuleLoader()) { auto &search_path_opts = GetCompilerInvocation().getSearchPathOptions(); std::unique_ptr esml_up = - swift::ExplicitSwiftModuleLoader::create( - *m_ast_context_up, m_dependency_tracker.get(), loading_mode, + LLDBExplicitSwiftModuleLoader::create( + *m_ast_context_up, m_cas.get(), m_action_cache.get(), + m_dependency_tracker.get(), loading_mode, search_path_opts.ExplicitSwiftModuleMapPath, search_path_opts.ExplicitSwiftModuleInputs, /*IgnoreSwiftSourceInfo*/ false); if (esml_up) { m_explicit_swift_module_loader = - static_cast(esml_up.get()); + static_cast(esml_up.get()); m_ast_context_up->addModuleLoader(std::move(esml_up), /*isClang=*/false, /*isDwarf=*/false, /*isInterface=*/false, @@ -3869,6 +3857,18 @@ ThreadSafeASTContext SwiftASTContext::GetASTContext() { } } + // 2. Create and install the memory buffer serialized module loader. + std::unique_ptr memory_buffer_loader_up( + swift::MemoryBufferSerializedModuleLoader::create( + *m_ast_context_up, m_dependency_tracker.get(), loading_mode, + /*IgnoreSwiftSourceInfo*/ false, /*BypassResilience*/ true)); + if (memory_buffer_loader_up) { + m_memory_buffer_module_loader = + static_cast( + memory_buffer_loader_up.get()); + m_ast_context_up->addModuleLoader(std::move(memory_buffer_loader_up)); + } + // Add a module interface checker. m_ast_context_up->addModuleInterfaceChecker( std::make_unique( @@ -9419,7 +9419,6 @@ bool SwiftASTContext::GetCompileUnitImportsImpl( *modules, Status &error) { // If EBM is enabled, disable implicit modules during contextual imports. - // fixme nullptr! bool turn_off_implicit = m_has_explicit_modules; auto reset = llvm::make_scope_exit([&] { if (turn_off_implicit) { @@ -9441,7 +9440,11 @@ bool SwiftASTContext::GetCompileUnitImportsImpl( // Swift. if (m_module_interface_loader) { auto &opts = m_module_interface_loader->getOptions(); - opts.disableImplicitSwiftModule = true; + // Turning this on would change the Clang module hash, which + // results in a "precompiled file '$HASH1/A.pcm' was compiled + // with module cache path '$HASH2', but the path is currently + // '$HASH1" when manually importing more modules. + opts.disableImplicitSwiftModule = false; opts.disableBuildingInterface = true; } // Clang. diff --git a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h index a0c6ae8e085ad..0bfe7d3d370bd 100644 --- a/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h +++ b/lldb/source/Plugins/TypeSystem/Swift/SwiftASTContext.h @@ -14,6 +14,7 @@ #define liblldb_SwiftASTContext_h_ #include "Plugins/LanguageRuntime/Swift/LockGuarded.h" +#include "Plugins/TypeSystem/Swift/LLDBExplicitModuleLoader.h" #include "Plugins/TypeSystem/Swift/TypeSystemSwift.h" #include "Plugins/TypeSystem/Swift/TypeSystemSwiftTypeRef.h" @@ -976,8 +977,8 @@ class SwiftASTContext : public TypeSystemSwift { /// Owned by the AST. swift::MemoryBufferSerializedModuleLoader *m_memory_buffer_module_loader = nullptr; - swift::ModuleLoader *m_explicit_swift_module_loader = nullptr; swift::ModuleInterfaceLoader *m_module_interface_loader = nullptr; + LLDBExplicitSwiftModuleLoader *m_explicit_swift_module_loader = nullptr; swift::ClangImporter *m_clangimporter = nullptr; /// Wraps the clang::ASTContext owned by ClangImporter. std::shared_ptr m_clangimporter_typesystem; diff --git a/lldb/test/API/lang/swift/clangimporter/caching/Makefile b/lldb/test/API/lang/swift/clangimporter/caching/Makefile index a0efe4cc196aa..322a8c253d2ec 100644 --- a/lldb/test/API/lang/swift/clangimporter/caching/Makefile +++ b/lldb/test/API/lang/swift/clangimporter/caching/Makefile @@ -2,4 +2,9 @@ SWIFT_SOURCES := main.swift SWIFT_ENABLE_EXPLICIT_MODULES := YES SWIFTFLAGS_EXTRAS = -I$(SRCDIR) -I/TEST_DIR -F/FRAMEWORK_DIR -cache-compile-job -cas-path $(BUILDDIR)/cas +all: a.out cas-config + include Makefile.rules + +cas-config: + echo "{\"CASPath\": \"$(BUILDDIR)/cas\"}" > .cas-config diff --git a/lldb/test/API/lang/swift/clangimporter/caching/TestSwiftClangImporterCaching.py b/lldb/test/API/lang/swift/clangimporter/caching/TestSwiftClangImporterCaching.py index ae094a9fafff6..0eeeacb65b06c 100644 --- a/lldb/test/API/lang/swift/clangimporter/caching/TestSwiftClangImporterCaching.py +++ b/lldb/test/API/lang/swift/clangimporter/caching/TestSwiftClangImporterCaching.py @@ -5,27 +5,23 @@ class TestSwiftClangImporterCaching(TestBase): - NO_DEBUG_INFO_TESTCASE = True - # Don't run ClangImporter tests if Clangimporter is disabled. @skipIf(setting=('symbols.use-swift-clangimporter', 'false')) @skipIf(setting=('symbols.swift-precise-compiler-invocation', 'false')) @skipUnlessDarwin @swiftTest def test(self): - """ - Test flipping on/off implicit modules. - """ self.build() lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec('main.swift')) log = self.getBuildArtifact("types.log") self.runCmd("settings set target.swift-clang-override-options +-DADDED=1") self.runCmd("settings set target.swift-extra-clang-flags -- -DEXTRA=1") - self.expect('log enable lldb types -f "%s"' % log) + self.expect('log enable lldb types symbols -f "%s"' % log) self.expect("expression obj", DATA_TYPES_DISPLAYED_CORRECTLY, substrs=["b ="]) self.filecheck('platform shell cat "%s"' % log, __file__) +# CHECK Loading module ClangB from CAS at llvmcas:// ### -cc1 should be round-tripped so there is no more `-cc1` in the extra args. Look for `-triple` which is a cc1 flag. # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift")::LogConfiguration() -- -triple ### Check include paths in the module are forwards. The first argument is the source directory. @@ -39,5 +35,4 @@ def test(self): # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift")::LogConfiguration() -- -DEXTRA=1 # CHECK: SwiftASTContextForExpressions(module: "a", cu: "main.swift") Module import remark: loaded module 'ClangA' # CHECK-NOT: -cc1 -# CHECK-NOT: -fmodule-file-cache-key -# CHECK-NOT: Clang error: +# CHECK-NOT: Clang error diff --git a/lldb/test/API/lang/swift/explicit_modules/implicit_fallback/TestSwiftExplicitModulesImplicitFallback.py b/lldb/test/API/lang/swift/explicit_modules/implicit_fallback/TestSwiftExplicitModulesImplicitFallback.py index f9acedd9fee22..ce5bdba88465c 100644 --- a/lldb/test/API/lang/swift/explicit_modules/implicit_fallback/TestSwiftExplicitModulesImplicitFallback.py +++ b/lldb/test/API/lang/swift/explicit_modules/implicit_fallback/TestSwiftExplicitModulesImplicitFallback.py @@ -32,11 +32,14 @@ def test_missing_explicit_modules(self): @swiftTest def test_sanity(self): """Check the normal behavior.""" - self.build() + mod_cache = self.getBuildArtifact("my-clang-modules-cache") + if os.path.isdir(mod_cache): + shutil.rmtree(mod_cache) - # This test verifies the case where explicit modules are missing. - # Remove explicit modules from their place in the module cache. - mod_cache = self.getBuildArtifact("private-module-cache") + self.runCmd('settings set symbols.clang-modules-cache-path "%s"' + % mod_cache) + self.runCmd("settings set symbols.swift-validate-typesystem false") + self.build() lldbutil.run_to_source_breakpoint( self, "Set breakpoint here", lldb.SBFileSpec("main.swift") @@ -46,7 +49,7 @@ def test_sanity(self): self.runCmd(f"log enable lldb types -f '{log}'") self.expect("expression c") - self.expect("expression -- import Foundation") + # FIXME: self.expect("expression -- import Foundation") self.filecheck(f"platform shell cat {log}", __file__, '--check-prefix=CHECK-SANITY') diff --git a/lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py b/lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py index 78c0c07110c7a..b5fb4348163b4 100644 --- a/lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py +++ b/lldb/test/API/lang/swift/explicit_modules/simple/TestSwiftExplicitModules.py @@ -41,6 +41,13 @@ def test_disable_esml(self): @skipUnlessDarwin def test_import(self): """Test an implicit import inside an explicit build""" + mod_cache = self.getBuildArtifact("my-clang-modules-cache") + if os.path.isdir(mod_cache): + shutil.rmtree(mod_cache) + + self.runCmd('settings set symbols.clang-modules-cache-path "%s"' + % mod_cache) + self.build() target, process, thread, bkpt = lldbutil.run_to_source_breakpoint( self, 'Set breakpoint here', lldb.SBFileSpec('main.swift')) diff --git a/lldb/test/Shell/Swift/caching.test b/lldb/test/Shell/Swift/caching.test index 2e0eaa5b4a670..3e6ffe56d12af 100644 --- a/lldb/test/Shell/Swift/caching.test +++ b/lldb/test/Shell/Swift/caching.test @@ -8,18 +8,14 @@ # RUN: -cache-compile-job -cas-path %t/cas -explicit-module-build \ # RUN: -module-name main -o %t/main -# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s +# RUN: sed "s|DIR|%/t|g" %t/lldb.script.template > %t/lldb.script +# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s --check-prefix=CAS-LOAD --check-prefix=CHECK -## Setup CAS and try loading from CAS -# RUN: sed "s|DIR|%/t|g" %t/lldb.script.template > %t/lldb_2.script -# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-LOAD --check-prefix=CHECK - -## Check fallback to file system. +## Check warning message # RUN: rm -rf %t/cas -# RUN: %lldb %t/main -s %t/lldb_2.script 2>&1 | FileCheck %s --check-prefix=CAS-FALLBACK --check-prefix=CHECK +# RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s --check-prefix=CAS-FALLBACK --check-prefix=CHECK -# CAS-FALLBACK: operator()() -- module '{{.*}}' cannot be load from CAS using key -# CAS-FALLBACK-SAME: fallback to load from file system +# CAS-FALLBACK: cannot be loaded from CAS using key: # CAS-LOAD: ConfigureCASStorage() -- Setup CAS from module list properties with cas path # CHECK: LogConfiguration() -- Extra clang arguments # CHECK-COUNT-1: LogConfiguration() -- -triple @@ -32,16 +28,6 @@ func test() { } test() -//--- lldb.script -# Force loading from interface to simulate no binary module available. -settings set symbols.swift-module-loading-mode prefer-interface -log enable lldb types -b test -run -# Create a SwiftASTContext -expr 1 -quit - //--- lldb.script.template # Force loading from interface to simulate no binary module available. settings set symbols.swift-module-loading-mode prefer-interface diff --git a/lldb/test/Shell/SymbolFile/cas-gmodule.test b/lldb/test/Shell/SymbolFile/cas-gmodule.test index fbc015209b384..f5ce4d3ce1b70 100644 --- a/lldb/test/Shell/SymbolFile/cas-gmodule.test +++ b/lldb/test/Shell/SymbolFile/cas-gmodule.test @@ -8,10 +8,10 @@ # RUN: sed "s|DIR|%/t|g" %t/cas-config.template > %t/.cas-config # RUN: %lldb %t/main -s %t/lldb.script 2>&1 | FileCheck %s -# FAIL: skip loading module 'Bottom' from CAS: CAS is not available +# FAIL: Failed to load module 'Bottom' from CAS: no CAS available # FAIL: Unable to locate module needed for external types. -# CHECK: loading module 'Bottom' using CASID -# CHECK: loading module 'Top' using CASID +# CHECK: Loading module 'Bottom' from CAS at +# CHECK: Loading module 'Top' from CAS at # CHECK: top = (x = 1) //--- module.modulemap diff --git a/llvm/include/llvm/CAS/CASConfiguration.h b/llvm/include/llvm/CAS/CASConfiguration.h index c07b7b546b7cf..b264004195b79 100644 --- a/llvm/include/llvm/CAS/CASConfiguration.h +++ b/llvm/include/llvm/CAS/CASConfiguration.h @@ -14,13 +14,15 @@ #ifndef LLVM_CAS_CASCONFIGURATION_H #define LLVM_CAS_CASCONFIGURATION_H +#include "llvm/ADT/Hashing.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/Support/Error.h" #include "llvm/Support/VirtualFileSystem.h" #include #include -namespace llvm::cas { +namespace llvm { +namespace cas { class ActionCache; class ObjectStore; @@ -72,8 +74,42 @@ class CASConfiguration { createFromSearchConfigFile( StringRef Path, llvm::IntrusiveRefCntPtr VFS = nullptr); + + /// DenseMap support \{ + static cas::CASConfiguration getDenseMapEmptyKey() { return {}; } + + static cas::CASConfiguration getDenseMapTombstoneKey() { + return {"$", "$", {}}; + } + unsigned getHashValue() const { + return detail::combineHashValue( + DenseMapInfo::getHashValue(StringRef(CASPath)), + DenseMapInfo::getHashValue(StringRef(PluginPath))); + } + /// \} + operator bool() { return !CASPath.empty() || !PluginPath.empty(); } }; -} // namespace llvm::cas +} // namespace cas + +template <> struct DenseMapInfo { + static cas::CASConfiguration getEmptyKey() { + return cas::CASConfiguration::getDenseMapEmptyKey(); + } + + static cas::CASConfiguration getTombstoneKey() { + return cas::CASConfiguration::getDenseMapTombstoneKey(); + } + + static unsigned getHashValue(const cas::CASConfiguration &config) { + return config.getHashValue(); + } + + static bool isEqual(const cas::CASConfiguration &LHS, + const cas::CASConfiguration &RHS) { + return LHS == RHS; + } +}; +} // namespace llvm #endif