lh123 updated this revision to Diff 229758.
lh123 added a comment.
update diff
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D70222/new/
https://reviews.llvm.org/D70222
Files:
clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
Index: clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
===================================================================
--- clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
+++ clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
@@ -17,8 +17,10 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/ConvertUTF.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
#include <string>
#include <tuple>
#include <vector>
@@ -27,6 +29,124 @@
namespace clangd {
namespace {
+bool expandResponseFile(llvm::StringRef FName, llvm::StringSaver &Saver,
+ llvm::cl::TokenizerCallback Tokenizer,
+ SmallVectorImpl<const char *> &NewArgv) {
+ llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MemBufOrErr =
+ llvm::MemoryBuffer::getFile(FName);
+ if (!MemBufOrErr)
+ return false;
+ llvm::MemoryBuffer &MemBuf = *MemBufOrErr.get();
+ StringRef Str(MemBuf.getBufferStart(), MemBuf.getBufferSize());
+
+ // If we have a UTF-16 byte order mark, convert to UTF-8 for parsing.
+ ArrayRef<char> BufRef(MemBuf.getBufferStart(), MemBuf.getBufferEnd());
+ std::string UTF8Buf;
+ // It is called byte order marker but the UTF-8 BOM is actually not affected
+ // by the host system's endianness.
+ auto HasUtF8ByteOrderMark = [](ArrayRef<char> S) {
+ return (S.size() >= 3 && S[0] == '\xef' && S[1] == '\xbb' &&
+ S[2] == '\xbf');
+ };
+ if (llvm::hasUTF16ByteOrderMark(BufRef)) {
+ if (!convertUTF16ToUTF8String(BufRef, UTF8Buf))
+ return false;
+ Str = StringRef(UTF8Buf);
+ }
+ // If we see UTF-8 BOM sequence at the beginning of a file, we shall remove
+ // these bytes before parsing.
+ // Reference: http://en.wikipedia.org/wiki/UTF-8#Byte_order_mark
+ else if (HasUtF8ByteOrderMark(BufRef))
+ Str = StringRef(BufRef.data() + 3, BufRef.size() - 3);
+ // Tokenize the contents into NewArgv.
+ Tokenizer(Str, Saver, NewArgv, false);
+ return true;
+}
+
+bool expandResponseFiles(tooling::CompileCommand &Cmd,
+ llvm::cl::TokenizerCallback Tokenizer) {
+ bool AllExpanded = true;
+ struct ResponseFileRecord {
+ llvm::StringRef File;
+ size_t End;
+ };
+ std::vector<std::string> &Argv = Cmd.CommandLine;
+ // To detect recursive response files, we maintain a stack of files and the
+ // position of the last argument in the file. This position is updated
+ // dynamically as we recursively expand files.
+ SmallVector<ResponseFileRecord, 3> FileStack;
+
+ // Push a dummy entry that represents the initial command line, removing
+ // the need to check for an empty list.
+ FileStack.push_back({"", Argv.size()});
+
+ // Don't cache Argv.size() because it can change.
+ for (unsigned I = 0; I != Argv.size();) {
+ while (I == FileStack.back().End) {
+ // Passing the end of a file's argument list, so we can remove it from the
+ // stack.
+ FileStack.pop_back();
+ }
+
+ std::string &Arg = Argv[I];
+
+ if (Arg[0] != '@') {
+ ++I;
+ continue;
+ }
+ SmallString<128> ResponseFile;
+ if (llvm::sys::path::is_relative(&Arg[1])) {
+ llvm::sys::path::append(ResponseFile, Cmd.Directory, &Arg[1]);
+ }
+ llvm::sys::path::remove_dots(ResponseFile);
+
+ auto IsEquivalent = [ResponseFile](const ResponseFileRecord &RFile) {
+ return llvm::sys::fs::equivalent(RFile.File, ResponseFile);
+ };
+
+ // Check for recursive response files.
+ if (std::any_of(FileStack.begin() + 1, FileStack.end(), IsEquivalent)) {
+ // This file is recursive, so we leave it in the argument stream and
+ // move on.
+ AllExpanded = false;
+ ++I;
+ continue;
+ }
+
+ // Replace this response file argument with the tokenization of its
+ // contents. Nested response files are expanded in subsequent iterations.
+ SmallVector<const char *, 0> ExpandedArgv;
+ llvm::BumpPtrAllocator Alloc;
+ llvm::StringSaver Saver(Alloc);
+ llvm::SmallVector<const char *, 64> T;
+ if (!expandResponseFile(ResponseFile, Saver, Tokenizer, ExpandedArgv)) {
+ // We couldn't read this file, so we leave it in the argument stream and
+ // move on.
+ AllExpanded = false;
+ ++I;
+ continue;
+ }
+
+ for (ResponseFileRecord &Record : FileStack) {
+ // Increase the end of all active records by the number of newly expanded
+ // arguments, minus the response file itself.
+ Record.End += ExpandedArgv.size() - 1;
+ }
+
+ FileStack.push_back({ResponseFile, I + ExpandedArgv.size()});
+ Argv.erase(Argv.begin() + I);
+ Argv.insert(Argv.begin() + I, ExpandedArgv.begin(), ExpandedArgv.end());
+ }
+
+ // If successful, the top of the file stack will mark the end of the Argv
+ // stream. A failure here indicates a bug in the stack popping logic above.
+ // Note that FileStack may have more than one element at this point because we
+ // don't have a chance to pop the stack when encountering recursive files at
+ // the end of the stream, so seeing that doesn't indicate a bug.
+ assert(FileStack.size() > 0 && Argv.size() == FileStack.back().End);
+ return AllExpanded;
+}
+
void adjustArguments(tooling::CompileCommand &Cmd,
llvm::StringRef ResourceDir) {
tooling::ArgumentsAdjuster ArgsAdjuster = tooling::combineAdjusters(
@@ -40,6 +160,14 @@
tooling::getClangSyntaxOnlyAdjuster()));
Cmd.CommandLine = ArgsAdjuster(Cmd.CommandLine, Cmd.Filename);
+
+ // ExpandResponseFile
+ auto Tokenizer = llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows()
+ ? llvm::cl::TokenizeWindowsCommandLine
+ : llvm::cl::TokenizeGNUCommandLine;
+ if (!expandResponseFiles(Cmd, Tokenizer))
+ log("Failed to expand response files for {0}", Cmd.Filename);
+
// Inject the resource dir.
// FIXME: Don't overwrite it if it's already there.
if (!ResourceDir.empty())
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
