[DependencyScan] Correct setup clang VFS for dependency scanning

Currently, dependency scanner is not reporting the redirecting files
that are baked inside swift-frontend for platform support. This causes
dependency scanner returns virtual path for those files, and
swift-driver/build-system will not be able to correct validate the files
on incremental build, causing incremental build to be almost clean
builds.

This behavior issue is caused by the dependency scanning file system
layer inside clang dependency scanner that caches stats. If the
redirecting files are created underneath the layer, the real path is
lost. This fixes the issue by moving the redirecting files above the
caching layer using `-ivfsoverlay` option.

In addition to that, this commit also unifies how clang importer and
clang dependency scanner initiate the VFS, making the logic much
simpler.

Author: cachemeifyoucan

include/swift/ClangImporter/ClangImporter.h

@@ -167,6 +167,8 @@ class ClangImporter final : public ClangModuleLoader {
   Implementation &Impl;
 
   bool requiresBuiltinHeadersInSystemModules = false;
+  bool needSystemVFSOverlay = false;
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseFS = nullptr;
 
   ClangImporter(ASTContext &ctx, DependencyTracker *tracker,
                 DWARFImporterDelegate *dwarfImporterDelegate);
@@ -205,6 +207,10 @@ class ClangImporter final : public ClangModuleLoader {
          DWARFImporterDelegate *dwarfImporterDelegate = nullptr,
          bool ignoreFileMapping = false);
 
+  static llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+  computeClangImporterFileSystem(const ASTContext &ctx, ClangImporter &importer,
+                                 bool suppressDiagnostics = false);
+
   std::vector<std::string>
   getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget = false);
 
@@ -531,6 +537,10 @@ class ClangImporter final : public ClangModuleLoader {
 
   std::string getClangModuleHash() const;
 
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getBaseFileSystem() const {
+    return baseFS;
+  }
+
   /// Get clang import creation cc1 args for swift explicit module build.
   std::vector<std::string> getSwiftExplicitModuleDirectCC1Args() const;
 
@@ -887,9 +897,8 @@ struct ClangInvocationFileMapping {
   /// Mapping from a file name to an existing file path.
   SmallVector<std::pair<std::string, std::string>, 2> redirectedFiles;
 
-  /// Mapping from a file name to a string of characters that represents the
-  /// contents of the file.
-  SmallVector<std::pair<std::string, std::string>, 1> overridenFiles;
+  /// MemoryBuffer that represents the file name and content to overload.
+  SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 1> overridenFiles;
 
   bool requiresBuiltinHeadersInSystemModules;
 };
@@ -924,15 +933,6 @@ ClangInvocationFileMapping getClangInvocationFileMapping(
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs = nullptr,
     bool suppressDiagnostic = false);
 
-/// Apply the given file mapping to the specified 'fileSystem', used
-/// primarily to inject modulemaps on platforms with non-modularized
-/// platform libraries.
-ClangInvocationFileMapping applyClangInvocationMapping(
-    const ClangInvocationFileMappingContext &ctx,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseVFS,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &fileSystem,
-    bool suppressDiagnostics = false);
-
 /// Information used to compute the access level of inherited C++ members.
 class ClangInheritanceInfo {
   /// The cumulative inheritance access specifier, that is used to compute the

lib/ClangImporter/ClangImporter.cpp

@@ -828,10 +828,9 @@ getEmbedBitcodeInvocationArguments(std::vector<std::string> &invocationArgStrs,
   });
 }
 
-void
-importer::addCommonInvocationArguments(
-    std::vector<std::string> &invocationArgStrs,
-    ASTContext &ctx, bool requiresBuiltinHeadersInSystemModules,
+void importer::addCommonInvocationArguments(
+    std::vector<std::string> &invocationArgStrs, ASTContext &ctx,
+    bool requiresBuiltinHeadersInSystemModules, bool needSystemVFSOverlay,
     bool ignoreClangTarget) {
   using ImporterImpl = ClangImporter::Implementation;
   llvm::Triple triple = ctx.LangOpts.Target;
@@ -1003,6 +1002,12 @@ importer::addCommonInvocationArguments(
     invocationArgStrs.push_back("-Xclang");
     invocationArgStrs.push_back("-fbuiltin-headers-in-system-modules");
   }
+
+  if (needSystemVFSOverlay) {
+    invocationArgStrs.push_back("-ivfsoverlay");
+    invocationArgStrs.push_back(
+        ClangImporter::Implementation::clangSystemVFSOverlayName);
+  }
 }
 
 bool ClangImporter::canReadPCH(StringRef PCHFilename) {
@@ -1151,6 +1156,70 @@ ClangImporter::getOrCreatePCH(const ClangImporterOptions &ImporterOptions,
   return PCHFilename.value();
 }
 
+llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>
+ClangImporter::computeClangImporterFileSystem(const ASTContext &ctx,
+                                              ClangImporter &importer,
+                                              bool suppressDiagnostics) {
+  assert(!ctx.CASOpts.HasImmutableFileSystem && "Use immutable file system");
+
+  ClangInvocationFileMapping fileMapping =
+      getClangInvocationFileMapping(ctx, nullptr, suppressDiagnostics);
+
+  auto importerOpts = ctx.ClangImporterOpts;
+  importer.requiresBuiltinHeadersInSystemModules =
+      fileMapping.requiresBuiltinHeadersInSystemModules;
+
+  auto fileSystem = llvm::vfs::getRealFileSystem();
+  if (!fileMapping.redirectedFiles.empty()) {
+    if (importerOpts.DumpClangDiagnostics) {
+      llvm::errs() << "clang importer redirected file mappings:\n";
+      for (const auto &mapping : fileMapping.redirectedFiles) {
+        llvm::errs() << "   mapping real file '" << mapping.second
+                     << "' to virtual file '" << mapping.first << "'\n";
+      }
+      llvm::errs() << "\n";
+    }
+    // Create a vfs overlay map for all redirects.
+    llvm::vfs::YAMLVFSWriter vfsWriter;
+    vfsWriter.setUseExternalNames(true);
+    for (const auto &mapping : fileMapping.redirectedFiles)
+      vfsWriter.addFileMapping(mapping.first, mapping.second);
+
+    std::string vfsYAML;
+    llvm::raw_string_ostream os(vfsYAML);
+    vfsWriter.write(os);
+
+    llvm::SmallString<256> overlayPath(ctx.SearchPathOpts.RuntimeResourcePath);
+    llvm::sys::path::append(overlayPath,
+                            Implementation::clangSystemVFSOverlayName);
+    fileMapping.overridenFiles.push_back(
+        llvm::MemoryBuffer::getMemBufferCopy(vfsYAML, overlayPath));
+
+    importer.needSystemVFSOverlay = true;
+  }
+
+  if (!fileMapping.overridenFiles.empty()) {
+    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> overridenVFS =
+        new llvm::vfs::InMemoryFileSystem();
+    for (auto &file : fileMapping.overridenFiles) {
+      if (importerOpts.DumpClangDiagnostics) {
+        llvm::errs() << "clang importer overriding file '"
+                     << file->getBufferIdentifier()
+                     << "' with the following contents:\n";
+        llvm::errs() << file->getBuffer() << "\n";
+      }
+      // Note MemoryBuffer is guaranteeed to be null-terminated.
+      overridenVFS->addFile(file->getBufferIdentifier(), 0, std::move(file));
+    }
+    llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> overlayVFS =
+        new llvm::vfs::OverlayFileSystem(fileSystem);
+    fileSystem = overlayVFS;
+    overlayVFS->pushOverlay(overridenVFS);
+  }
+
+  return fileSystem;
+}
+
 std::vector<std::string>
 ClangImporter::getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget) {
   assert(!ctx.ClangImporterOpts.DirectClangCC1ModuleBuild &&
@@ -1169,7 +1238,8 @@ ClangImporter::getClangDriverArguments(ASTContext &ctx, bool ignoreClangTarget)
     break;
   }
   addCommonInvocationArguments(invocationArgStrs, ctx,
-      requiresBuiltinHeadersInSystemModules, ignoreClangTarget);
+                               requiresBuiltinHeadersInSystemModules,
+                               needSystemVFSOverlay, ignoreClangTarget);
   return invocationArgStrs;
 }
 
@@ -1235,6 +1305,10 @@ std::optional<std::vector<std::string>> ClangImporter::getClangCC1Arguments(
         CI->getTargetOpts().DarwinTargetVariantTriple = ctx.LangOpts.TargetVariant->str();
     }
 
+    if (needSystemVFSOverlay)
+      CI->getHeaderSearchOpts().AddVFSOverlayFile(
+          Implementation::clangSystemVFSOverlayName);
+
     // Forward the index store path. That information is not passed to scanner
     // and it is cached invariant so we don't want to re-scan if that changed.
     CI->getFrontendOpts().IndexStorePath = ctx.ClangImporterOpts.IndexStorePath;
@@ -1348,18 +1422,21 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
     }
   }
 
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS =
-      ctx.SourceMgr.getFileSystem();
+  // Configure ClangImporter file system. There are two situations:
+  // * If caching is used, thus file system is immutable, the one immutable file
+  //   system is shared between swift frontend and ClangImporter.
+  // * Otherwise, ClangImporter file system is configure from scratch from
+  //   RealFileSystem using ivfsoverlay options.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> vfs =
+      ctx.CASOpts.HasImmutableFileSystem
+          ? ctx.SourceMgr.getFileSystem()
+          : computeClangImporterFileSystem(ctx, *importer, ignoreFileMapping);
 
-  ClangInvocationFileMapping fileMapping =
-    applyClangInvocationMapping(ctx, nullptr, VFS, ignoreFileMapping);
-
-  importer->requiresBuiltinHeadersInSystemModules =
-      fileMapping.requiresBuiltinHeadersInSystemModules;
+  importer->baseFS = vfs;
 
   // Create a new Clang compiler invocation.
   {
-    if (auto ClangArgs = importer->getClangCC1Arguments(ctx, VFS))
+    if (auto ClangArgs = importer->getClangCC1Arguments(ctx, vfs))
       importer->Impl.ClangArgs = *ClangArgs;
     else
       return nullptr;
@@ -1373,7 +1450,7 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
       llvm::errs() << "'\n";
     }
     importer->Impl.Invocation = createClangInvocation(
-        importer.get(), importerOpts, VFS, importer->Impl.ClangArgs);
+        importer.get(), importerOpts, vfs, importer->Impl.ClangArgs);
     if (!importer->Impl.Invocation)
       return nullptr;
   }
@@ -1424,7 +1501,7 @@ std::unique_ptr<ClangImporter> ClangImporter::create(
   auto actualDiagClient = std::make_unique<ClangDiagnosticConsumer>(
       importer->Impl, instance.getDiagnosticOpts(),
       importerOpts.DumpClangDiagnostics);
-  instance.createVirtualFileSystem(std::move(VFS), actualDiagClient.get());
+  instance.createVirtualFileSystem(std::move(vfs), actualDiagClient.get());
   instance.createFileManager();
   instance.createDiagnostics(actualDiagClient.release());
 

lib/ClangImporter/ClangIncludePaths.cpp

@@ -450,26 +450,26 @@ static void getLibStdCxxFileMapping(
     includeHeaderInModuleMap(additionalFile);
   os << contents.substr(headerInjectionPoint);
 
-  fileMapping.overridenFiles.push_back(
-      {std::string(injectedModuleMapPath), std::move(os.str())});
+  fileMapping.overridenFiles.push_back(llvm::MemoryBuffer::getMemBufferCopy(
+      additionalHeaderDirectives, injectedModuleMapPath));
 }
 
 namespace {
-std::string
-GetPlatformAuxiliaryFile(StringRef Platform, StringRef File,
-                         const SearchPathOptions &Options) {
+std::string GetPlatformAuxiliaryFile(StringRef Platform, StringRef File,
+                                     llvm::vfs::FileSystem &VFS,
+                                     const SearchPathOptions &Options) {
   StringRef SDKPath = Options.getSDKPath();
   if (!SDKPath.empty()) {
     llvm::SmallString<261> path{SDKPath};
     llvm::sys::path::append(path, "usr", "share", File);
-    if (llvm::sys::fs::exists(path))
+    if (VFS.exists(path))
       return path.str().str();
   }
 
   if (!Options.RuntimeResourcePath.empty()) {
     llvm::SmallString<261> path{Options.RuntimeResourcePath};
     llvm::sys::path::append(path, Platform, File);
-    if (llvm::sys::fs::exists(path))
+    if (VFS.exists(path))
       return path.str().str();
   }
 
@@ -514,7 +514,7 @@ void GetWindowsFileMappings(
     llvm::sys::path::append(WinSDKInjection, "module.modulemap");
 
     AuxiliaryFile = GetPlatformAuxiliaryFile("windows", "winsdk_um.modulemap",
-                                             SearchPathOpts);
+                                             VFS, SearchPathOpts);
     if (!AuxiliaryFile.empty())
       fileMapping.redirectedFiles.emplace_back(std::string(WinSDKInjection),
                                                AuxiliaryFile);
@@ -523,7 +523,7 @@ void GetWindowsFileMappings(
     llvm::sys::path::remove_filename(WinSDKInjection);
     llvm::sys::path::append(WinSDKInjection, "shared", "module.modulemap");
     AuxiliaryFile = GetPlatformAuxiliaryFile(
-        "windows", "winsdk_shared.modulemap", SearchPathOpts);
+        "windows", "winsdk_shared.modulemap", VFS, SearchPathOpts);
     if (!AuxiliaryFile.empty())
       fileMapping.redirectedFiles.emplace_back(std::string(WinSDKInjection),
                                                AuxiliaryFile);
@@ -540,8 +540,8 @@ void GetWindowsFileMappings(
     llvm::sys::path::append(UCRTInjection, "Include", UCRTSDK.Version, "ucrt");
     llvm::sys::path::append(UCRTInjection, "module.modulemap");
 
-    AuxiliaryFile =
-        GetPlatformAuxiliaryFile("windows", "ucrt.modulemap", SearchPathOpts);
+    AuxiliaryFile = GetPlatformAuxiliaryFile("windows", "ucrt.modulemap", VFS,
+                                             SearchPathOpts);
     if (!AuxiliaryFile.empty()) {
       // The ucrt module map has the C standard library headers all together.
       // That leads to module cycles with the clang _Builtin_ modules. e.g.
@@ -578,18 +578,16 @@ void GetWindowsFileMappings(
     llvm::sys::path::append(VCToolsInjection, "include");
 
     llvm::sys::path::append(VCToolsInjection, "module.modulemap");
-    AuxiliaryFile =
-        GetPlatformAuxiliaryFile("windows", "vcruntime.modulemap",
-                                 SearchPathOpts);
+    AuxiliaryFile = GetPlatformAuxiliaryFile("windows", "vcruntime.modulemap",
+                                             VFS, SearchPathOpts);
     if (!AuxiliaryFile.empty())
       fileMapping.redirectedFiles.emplace_back(std::string(VCToolsInjection),
                                                AuxiliaryFile);
 
     llvm::sys::path::remove_filename(VCToolsInjection);
     llvm::sys::path::append(VCToolsInjection, "vcruntime.apinotes");
-    AuxiliaryFile =
-        GetPlatformAuxiliaryFile("windows", "vcruntime.apinotes",
-                                 SearchPathOpts);
+    AuxiliaryFile = GetPlatformAuxiliaryFile("windows", "vcruntime.apinotes",
+                                             VFS, SearchPathOpts);
     if (!AuxiliaryFile.empty())
       fileMapping.redirectedFiles.emplace_back(std::string(VCToolsInjection),
                                                AuxiliaryFile);
@@ -610,9 +608,9 @@ void GetWindowsFileMappings(
     for (const char * const header : kInjectedHeaders) {
       llvm::sys::path::remove_filename(VCToolsInjection);
       llvm::sys::path::append(VCToolsInjection, header);
-      if (!llvm::sys::fs::exists(VCToolsInjection))
-        fileMapping.overridenFiles.emplace_back(std::string{VCToolsInjection},
-                                                "");
+      if (!VFS.exists(VCToolsInjection))
+        fileMapping.overridenFiles.emplace_back(
+            llvm::MemoryBuffer::getMemBufferCopy("", VCToolsInjection));
     }
   }
 }
@@ -695,49 +693,3 @@ ClangInvocationFileMapping swift::getClangInvocationFileMapping(
                          result.requiresBuiltinHeadersInSystemModules);
   return result;
 }
-
-ClangInvocationFileMapping swift::applyClangInvocationMapping(
-    const ClangInvocationFileMappingContext &ctx,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> baseVFS,
-    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> &fileSystem,
-    bool suppressDiagnostics) {
-  if (ctx.CASOpts.HasImmutableFileSystem)
-    return ClangInvocationFileMapping();
-
-  ClangInvocationFileMapping fileMapping =
-    getClangInvocationFileMapping(ctx, baseVFS, suppressDiagnostics);
-
-  auto importerOpts = ctx.ClangImporterOpts;
-  // Wrap Swift's FS to allow Clang to override the working directory
-  fileSystem = llvm::vfs::RedirectingFileSystem::create(
-      fileMapping.redirectedFiles, true, *fileSystem);
-  if (importerOpts.DumpClangDiagnostics) {
-    llvm::errs() << "clang importer redirected file mappings:\n";
-    for (const auto &mapping : fileMapping.redirectedFiles) {
-      llvm::errs() << "   mapping real file '" << mapping.second
-                   << "' to virtual file '" << mapping.first << "'\n";
-    }
-    llvm::errs() << "\n";
-  }
-
-  if (!fileMapping.overridenFiles.empty()) {
-    llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> overridenVFS =
-        new llvm::vfs::InMemoryFileSystem();
-    for (const auto &file : fileMapping.overridenFiles) {
-      if (importerOpts.DumpClangDiagnostics) {
-        llvm::errs() << "clang importer overriding file '" << file.first
-                     << "' with the following contents:\n";
-        llvm::errs() << file.second << "\n";
-      }
-      // Note MemoryBuffer is guaranteeed to be null-terminated.
-      overridenVFS->addFile(file.first, 0,
-                            llvm::MemoryBuffer::getMemBufferCopy(file.second));
-    }
-    llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> overlayVFS =
-        new llvm::vfs::OverlayFileSystem(fileSystem);
-    fileSystem = overlayVFS;
-    overlayVFS->pushOverlay(overridenVFS);
-  }
-
-  return fileMapping;
-}