llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clangd Author: Access (Decodetalkers) <details> <summary>Changes</summary> This pr will work after you compile the project with gcc once a time. I still need to tidy up the debug information --- Patch is 31.99 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/200001.diff 12 Files Affected: - (modified) clang-tools-extra/clangd/ClangdLSPServer.cpp (+1) - (modified) clang-tools-extra/clangd/ClangdServer.cpp (+1) - (modified) clang-tools-extra/clangd/Compiler.cpp (+2) - (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.cpp (+16-2) - (modified) clang-tools-extra/clangd/GlobalCompilationDatabase.h (+1-1) - (modified) clang-tools-extra/clangd/ModulesBuilder.cpp (+41-1) - (modified) clang-tools-extra/clangd/ModulesBuilder.h (+2-1) - (modified) clang-tools-extra/clangd/ParsedAST.cpp (+2) - (modified) clang-tools-extra/clangd/Preamble.cpp (+4) - (modified) clang-tools-extra/clangd/ProjectModules.cpp (+208-2) - (modified) clang-tools-extra/clangd/TUScheduler.cpp (+8) - (modified) clang/lib/Tooling/JSONCompilationDatabase.cpp (+34-23) ``````````diff diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp index 04f58ab6446d1..f52d46248c4cf 100644 --- a/clang-tools-extra/clangd/ClangdLSPServer.cpp +++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -567,6 +567,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params, std::move(Mangler)); if (Opts.EnableExperimentalModulesSupport) { + // Here build a ModulesManager ModulesManager.emplace(*CDB); Opts.ModulesManager = &*ModulesManager; } diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp index 9c9290b8b6b1b..e01e8f5752f8a 100644 --- a/clang-tools-extra/clangd/ClangdServer.cpp +++ b/clang-tools-extra/clangd/ClangdServer.cpp @@ -230,6 +230,7 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB, Transient(Opts.ImplicitCancellation ? TUScheduler::InvalidateOnUpdate : TUScheduler::NoInvalidation), DirtyFS(std::make_unique<DraftStoreFS>(TFS, DraftMgr)) { + // NOTE: main logic here if (Opts.AsyncThreadsCount != 0) IndexTasks.emplace(); // Pass a callback into `WorkScheduler` to extract symbols from a newly diff --git a/clang-tools-extra/clangd/Compiler.cpp b/clang-tools-extra/clangd/Compiler.cpp index 9ea7df139382a..1651232878051 100644 --- a/clang-tools-extra/clangd/Compiler.cpp +++ b/clang-tools-extra/clangd/Compiler.cpp @@ -92,6 +92,7 @@ void disableUnsupportedOptions(CompilerInvocation &CI) { CI.getLangOpts().XRayNeverInstrumentFiles.clear(); } +// FIXME: main problem is here std::unique_ptr<CompilerInvocation> buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, std::vector<std::string> *CC1Args) { @@ -127,6 +128,7 @@ buildCompilerInvocation(const ParseInputs &Inputs, clang::DiagnosticConsumer &D, return CI; } +// So the error is from here std::unique_ptr<CompilerInstance> prepareCompilerInstance(std::unique_ptr<clang::CompilerInvocation> CI, const PrecompiledPreamble *Preamble, diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp index adb771ecbbaad..4b2a32a84f1ee 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -269,6 +269,7 @@ parseFixed(PathRef Path, llvm::StringRef Data, std::string &Error) { llvm::sys::path::parent_path(Path), Data, Error); } +// NOTE: Core bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load( llvm::vfs::FileSystem &FS) { dlog("Probing directory {0}", Path); @@ -309,11 +310,19 @@ bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load( return true; case CachedFile::LoadResult::FoundNewData: // We have a new CDB! + // NOTE: this time the --driver is still not be added CDB = Entry.Parser(Entry.File->Path, Loaded.Buffer->getBuffer(), Error); - if (CDB) + if (CDB) { log("{0} compilation database from {1}", Active ? "Reloaded" : "Loaded", Entry.File->Path); - else + elog(" =========== the command len ==============, {0}", CDB->getAllCompileCommands().size()); + auto commmand_0 = CDB->getAllCompileCommands()[0]; + std::string a_command; + for(auto &command : commmand_0.CommandLine) { + a_command += " " + command; + } + elog(" =========== the command {0} ==============", a_command); + } else elog("Failed to load compilation database from {0}: {1}", Entry.File->Path, Error); ActiveCachedFile = Entry.File; @@ -771,6 +780,7 @@ DirectoryBasedGlobalCompilationDatabase::getProjectModules(PathRef File) const { if (!Res) return {}; + // NOTE: 3 return clang::clangd::getProjectModules(Res->CDB, Opts.TFS); } @@ -811,6 +821,7 @@ OverlayCDB::getCompileCommand(PathRef File) const { if (!Cmd) return std::nullopt; if (Mangler) + // NOTE: from here, the command line changed forever Mangler(*Cmd, File); return Cmd; } @@ -848,6 +859,7 @@ bool OverlayCDB::setCompileCommand(PathRef File, return true; } +// this part seems important std::unique_ptr<ProjectModules> OverlayCDB::getProjectModules(PathRef File) const { auto MDB = DelegatingCDB::getProjectModules(File); @@ -879,6 +891,7 @@ DelegatingCDB::DelegatingCDB( BaseOwner = std::move(Base); } +// Base is always DirectoryBasedGlobalCompilationDatabase std::optional<tooling::CompileCommand> DelegatingCDB::getCompileCommand(PathRef File) const { if (!Base) @@ -892,6 +905,7 @@ std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const { return Base->getProjectInfo(File); } +// CompoundProjectModules std::unique_ptr<ProjectModules> DelegatingCDB::getProjectModules(PathRef File) const { if (!Base) diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.h b/clang-tools-extra/clangd/GlobalCompilationDatabase.h index 415c7f50f8606..9cff6a310d4ff 100644 --- a/clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -201,7 +201,7 @@ getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs); class OverlayCDB : public DelegatingCDB { public: // Makes adjustments to a tooling::CompileCommand which will be used to - // process a file (possibly different from the one in the command). + // ---driver-mode-driver-modeprocess a file (possibly different from the one in the command). using CommandMangler = llvm::unique_function<void(tooling::CompileCommand &, StringRef File) const>; diff --git a/clang-tools-extra/clangd/ModulesBuilder.cpp b/clang-tools-extra/clangd/ModulesBuilder.cpp index 706fd459e15ec..da35d2a814ca8 100644 --- a/clang-tools-extra/clangd/ModulesBuilder.cpp +++ b/clang-tools-extra/clangd/ModulesBuilder.cpp @@ -472,6 +472,10 @@ class ReusablePrerequisiteModules : public PrerequisiteModules { ~ReusablePrerequisiteModules() override = default; + // NOTE: problem is here + // We should make gcc version + // FIXME: try to under stand if it will work or not + // And What the Fuck? it uses clang++ void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override { // Appending all built module files. for (const auto &RequiredModule : RequiredModules) @@ -600,6 +604,9 @@ buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName, }); (void)RemoveTemporaryModuleFile; + // NOTE: but the output should be o file not the pcm file + // What has it done? + // FIXME: ??? First I should make it clear that the command is Cmd.Output = TemporaryModuleFilePath.str().str(); ParseInputs Inputs; @@ -625,6 +632,7 @@ buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName, // BMI files. CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true; + // WHAT IS THE OPTIONS? BuiltModuleFiles.adjustHeaderSearchOptions(CI->getHeaderSearchOpts()); CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output; @@ -645,12 +653,13 @@ buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName, Cmds += Arg; } + // FIXME: let it compile clangd::vlog("Failed to compile {0} with command: {1}", ModuleUnitFileName, Cmds); std::string BuiltModuleFilesStr = BuiltModuleFiles.getAsString(); if (!BuiltModuleFilesStr.empty()) - clangd::vlog("The actual used module files built by clangd is {0}", + clangd::elog("The actual used module files built by clangd is {0}", BuiltModuleFilesStr); return llvm::createStringError( @@ -862,6 +871,7 @@ class CachingProjectModules : public ProjectModules { } std::vector<std::string> getRequiredModules(PathRef File) override { + elog("========= get module here ======"); return MDB->getRequiredModules(File); } @@ -873,10 +883,12 @@ class CachingProjectModules : public ProjectModules { return MDB->getModuleNameState(ModuleName); } + // NOTE: then it enter here std::string getSourceForModuleName(llvm::StringRef ModuleName, PathRef RequiredSrcFile) override { auto ModuleState = MDB->getModuleNameState(ModuleName); + elog("Scan start"); if (ModuleState == ModuleNameState::Multiple) { std::string CachedResult = Cache.getMultipleSourceForModuleName(ModuleName, RequiredSrcFile); @@ -884,6 +896,8 @@ class CachingProjectModules : public ProjectModules { // Verify Cached Result by seeing if the source declaring the same module // as we query. if (!CachedResult.empty()) { + // NOTE: MDB is ScanningAllProjectModules + std::string ModuleNameOfCachedSource = MDB->getModuleNameForSource(CachedResult); if (ModuleNameOfCachedSource == ModuleName) @@ -893,6 +907,9 @@ class CachingProjectModules : public ProjectModules { Cache.eraseMultipleEntry(ModuleName, RequiredSrcFile); } + // One of the is CompoundProjectModules + // So understand the logic will be enough + // NOTE: this logic is fixed auto Result = MDB->getSourceForModuleName(ModuleName, RequiredSrcFile); if (!Result.empty()) Cache.addMultipleEntry(ModuleName, RequiredSrcFile, Result); @@ -1059,6 +1076,7 @@ void ModulesBuilder::ModulesBuilderImpl:: CacheRoot); } +// I need to add the data here instead go to the next logic void ModulesBuilder::ModulesBuilderImpl::getPrebuiltModuleFile( StringRef ModuleName, PathRef ModuleUnitFileName, const ThreadsafeFS &TFS, ReusablePrerequisiteModules &BuiltModuleFiles) { @@ -1103,14 +1121,20 @@ void ModulesBuilder::ModulesBuilderImpl::getPrebuiltModuleFile( } } +// NOTE Problem is the BuiltModuleFiles +// NOTE: so it is the first place it start scanning +// FIXME: cannot use gcc.gcm llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile( PathRef RequiredSource, StringRef ModuleName, const ThreadsafeFS &TFS, CachingProjectModules &MDB, ReusablePrerequisiteModules &BuiltModuleFiles) { if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName)) return llvm::Error::success(); + elog("============== scanning starts from here? ============="); std::string ModuleUnitFileName = MDB.getSourceForModuleName(ModuleName, RequiredSource); + + elog("====== so we got gcc module source {0}, file is : {1}", ModuleName, ModuleUnitFileName); /// It is possible that we're meeting third party modules (modules whose /// source are not in the project. e.g, the std module may be a third-party /// module for most project) or something wrong with the implementation of @@ -1129,12 +1153,20 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile( // Get Required modules in topological order. auto ReqModuleNames = getAllRequiredModules(RequiredSource, MDB, ModuleName); + // FIXME: first I should print it once a time + // NOTE: Ok, seems we can get the right module name, also , we can get its source. + // Then let's do it + // We should not reuse the logic of clangd, This time we can get the Cmd, so we can also know if it is gcc for (llvm::StringRef ReqModuleName : ReqModuleNames) { + // NOTE: ohh. I have print it before! + elog("ReqModuleName: {0}", ReqModuleName.str()); if (BuiltModuleFiles.isModuleUnitBuilt(ReqModuleName)) continue; std::string ReqFileName = MDB.getSourceForModuleName(ReqModuleName, RequiredSource); + // FIXME: problem is.. the compile command for gcc will never work + // So we need a translation auto Cmd = getCDB().getCompileCommand(ReqFileName); if (!Cmd) return llvm::createStringError( @@ -1144,6 +1176,7 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile( garbageCollectModuleCacheForProjectRoot(PI->SourceRoot); const std::string CommandHash = getCompileCommandStringHash(*Cmd); + // NOTE: this logic is for clang++, so We should never use it const std::string PublishedModuleFilePath = getPublishedModuleFilePath( ReqModuleName, getModuleFilesDirectory(ReqFileName, *Cmd, getCDB())); @@ -1219,6 +1252,8 @@ llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile( } bool ModulesBuilder::hasRequiredModules(PathRef File) { + // NOTE: the right logic is in DirectoryBasedGlobalCompilationDatabase + // Now it is CompoundProjectModules std::unique_ptr<ProjectModules> MDB = Impl->getCDB().getProjectModules(File); if (!MDB) return false; @@ -1228,10 +1263,14 @@ bool ModulesBuilder::hasRequiredModules(PathRef File) { return !CachedMDB.getRequiredModules(File).empty(); } +// NOTE: so we finally got to the position +// Maybe we can just create a gcc version std::unique_ptr<PrerequisiteModules> ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, const ThreadsafeFS &TFS) { + // NOTE: them MDB always is the DirectoryBasedGlobalCompilationDatabase std::unique_ptr<ProjectModules> MDB = Impl->getCDB().getProjectModules(File); + elog("Enter here? ====================="); if (!MDB) { elog("Failed to get Project Modules information for {0}", File); return std::make_unique<FailedPrerequisiteModules>(); @@ -1244,6 +1283,7 @@ ModulesBuilder::buildPrerequisiteModulesFor(PathRef File, if (RequiredModuleNames.empty()) return std::make_unique<ReusablePrerequisiteModules>(); + // NOTE: Seems we need to change here auto RequiredModules = std::make_unique<ReusablePrerequisiteModules>(); for (llvm::StringRef RequiredModuleName : RequiredModuleNames) { // Return early if there is any error. diff --git a/clang-tools-extra/clangd/ModulesBuilder.h b/clang-tools-extra/clangd/ModulesBuilder.h index b0e110b92b6a7..1490fa51d401c 100644 --- a/clang-tools-extra/clangd/ModulesBuilder.h +++ b/clang-tools-extra/clangd/ModulesBuilder.h @@ -81,10 +81,11 @@ class PrerequisiteModules { /// This class handles building module files for a given source file. /// -/// In the future, we want the class to manage the module files acorss +/// In the future, we want the class to manage the module files across /// different versions and different source files. class ModulesBuilder { public: + // Always use the DirectoryBasedGlobalCompilationDatabase ModulesBuilder(const GlobalCompilationDatabase &CDB); ~ModulesBuilder(); diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp index e2a49f384a3e9..c84871963e1c1 100644 --- a/clang-tools-extra/clangd/ParsedAST.cpp +++ b/clang-tools-extra/clangd/ParsedAST.cpp @@ -463,6 +463,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs, Patch = PreamblePatch::createFullPatch(Filename, Inputs, *Preamble); Patch->apply(*CI); } + + // NOTE: and they tried to build auto Clang = prepareCompilerInstance( std::move(CI), Inputs.Opts.SkipPreambleBuild ? nullptr : PreamblePCH, llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS, diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp index 58da7edcf3b93..aa555938ffb9a 100644 --- a/clang-tools-extra/clangd/Preamble.cpp +++ b/clang-tools-extra/clangd/Preamble.cpp @@ -569,6 +569,7 @@ class DiagPatcher { }; } // namespace +// NOTE: here did we enter here? std::shared_ptr<const PreambleData> buildPreamble(PathRef FileName, CompilerInvocation CI, const ParseInputs &Inputs, bool StoreInMemory, @@ -584,6 +585,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, trace::Span Tracer("BuildPreamble"); SPAN_ATTACH(Tracer, "File", FileName); std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners; + // NOTE: ok seems we got it? if (Inputs.FeatureModules) { for (auto &M : *Inputs.FeatureModules) { if (auto Listener = M.astListeners()) @@ -676,6 +678,8 @@ buildPreamble(PathRef FileName, CompilerInvocation CI, if (Inputs.ModulesManager) { WallTimer PrerequisiteModuleTimer; PrerequisiteModuleTimer.startTimer(); + // NOTE: It is the real enter point 2 =========== + // NOTE: it failed to get the RequiredModules Result->RequiredModules = Inputs.ModulesManager->buildPrerequisiteModulesFor(FileName, *Inputs.TFS); diff --git a/clang-tools-extra/clangd/ProjectModules.cpp b/clang-tools-extra/clangd/ProjectModules.cpp index d3727171bff12..4e856999a982a 100644 --- a/clang-tools-extra/clangd/ProjectModules.cpp +++ b/clang-tools-extra/clangd/ProjectModules.cpp @@ -12,11 +12,16 @@ #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/Tooling/DependencyScanningTool.h" #include "clang/Tooling/Tooling.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FileUtilities.h" #include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Regex.h" #include "llvm/TargetParser/Host.h" namespace clang::clangd { @@ -171,6 +176,8 @@ class ModuleDependencyScanner { /// Scanning the single file specified by \param FilePath. std::optional<ModuleDependencyInfo> scan(PathRef FilePath, const ProjectModules::CommandMangler &Mangler); + std::optional<ModuleDependencyInfo> scanGcc(tooling::CompileCommand Cmd, + PathRef MapFile); /// Scanning every source file in the current project to get the /// <module-name> to <module-unit-source> map. @@ -209,15 +216,214 @@ class ModuleDependencyScanner { llvm::StringMap<std::string> ModuleNameToSource; }; +namespace gcc { +static const llvm::Regex ImportRegex = + llvm::Regex("import: ([^ ]*) ([^ ]*.gcm)"); +static const llvm::Regex ModuleRegex = llvm::Regex("module: ([^ ]*)"); +static const llvm::Regex SourceRegex = llvm::Regex("source: ([^ ]*)"); +static const llvm::Regex CwdRegex = llvm::Regex("cwd: ([^ ]*)"); + +static const llvm::Regex ModmapRegex = llvm::Regex("([^ ^$^\n]*) ([^ ]*.gcm)"); + +struct RoadMapInfo { + std::string Name; + std::string Path; +}; + +struct ReadElfInfo { + std::string Source; + std::string ModuleName; + std::vector<std::string> Imports; + + static std::optional<ReadElfInfo> get(llvm::StringRef Source); +}; + +std::optional<ReadElfInfo> ReadElfInfo::get(llvm::StringRef Content) { + std::vector<std::string> Imports = {}; + std::string Source; + std::string ModuleName; + std::string Cwd; + { + llvm::StringRef CwdText = Content; + llvm::SmallVector<llvm::StringRef, 1> Matches; + std::string Error; + if (!CwdRegex.match(CwdText, &Matches, &Error)) { + return std::nullopt; + } + Cwd = Matches[1].trim().str(); + } + { + llvm::StringRef ImportText = Content; + while (!ImportText.empty()) { + llvm::SmallVector<llvm::StringRef, 2> Matches; + std::string Error; + if (!ImportRegex.match(ImportText, &Matches, &Error)) { + break; + } + + auto ImportModule = Matches[1].trim().str(); + Imports.push_back(ImportModule); + size_t Pos = ImportText.find(Matches[0]); + ImportText = ImportText.drop_front(Pos + Matches[0].size()); + } + } + + { + llvm::StringRef SourceText = Content; + llvm::SmallVector<llvm::StringRef, 1> Matches; + std::string Error; + if (!SourceRegex.match(SourceText, &Matches, &Error)) { + return std::nullopt; + } + + llvm::StringRef SourcePa = Matches[1].trim(); + if (llvm::sys::path::is_absolute(SourcePa)) { + Source = SourcePa.str(); + + } else { + llvm::StringRef PathRef = Cwd; + llvm::SmallString<128> CurrentPath = PathRef; + llvm::sys::path::append(CurrentPath, SourcePa); + Source = CurrentPath.str(); + } + } + { + llvm::StringRef ModuleText = Content; + llvm::SmallVector<llvm::StringRef, 1> Matches; + std::string Error; + if (!ModuleRegex.match(ModuleText, &Matches, &Error)) { + return std::nullopt; + } + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/200001 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
