gamesh411 updated this revision to Diff 257622.
gamesh411 added a comment.
Reorganize test code to overcome testing infrastructure limitations
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D75665/new/
https://reviews.llvm.org/D75665
Files:
clang/include/clang/CrossTU/CrossTranslationUnit.h
clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
clang/lib/CrossTU/CMakeLists.txt
clang/lib/CrossTU/CrossTranslationUnit.cpp
clang/lib/Frontend/CompilerInvocation.cpp
clang/lib/StaticAnalyzer/Core/CallEvent.cpp
clang/test/Analysis/Inputs/ctu-other.c
clang/test/Analysis/Inputs/ctu-other.c.externalDefMap.ast-dump.txt
clang/test/Analysis/Inputs/ctu-other.c.externalDefMap.txt
clang/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt
clang/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.txt
clang/test/Analysis/analyzer-config.c
clang/test/Analysis/ctu-different-triples.cpp
clang/test/Analysis/ctu-main.c
clang/test/Analysis/ctu-main.cpp
clang/test/Analysis/ctu-on-demand-parsing.c
clang/test/Analysis/ctu-on-demand-parsing.cpp
clang/test/Analysis/ctu-unknown-parts-in-triples.cpp
clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
Index: clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
===================================================================
--- clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
+++ clang/unittests/CrossTU/CrossTranslationUnitTest.cpp
@@ -7,10 +7,11 @@
//===----------------------------------------------------------------------===//
#include "clang/CrossTU/CrossTranslationUnit.h"
-#include "clang/Frontend/CompilerInstance.h"
#include "clang/AST/ASTConsumer.h"
+#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/FrontendAction.h"
#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/ToolOutputFile.h"
@@ -162,7 +163,7 @@
IndexFile.os().flush();
EXPECT_TRUE(llvm::sys::fs::exists(IndexFileName));
llvm::Expected<llvm::StringMap<std::string>> IndexOrErr =
- parseCrossTUIndex(IndexFileName, "");
+ parseCrossTUIndex(IndexFileName);
EXPECT_TRUE((bool)IndexOrErr);
llvm::StringMap<std::string> ParsedIndex = IndexOrErr.get();
for (const auto &E : Index) {
@@ -173,25 +174,5 @@
EXPECT_TRUE(Index.count(E.getKey()));
}
-TEST(CrossTranslationUnit, CTUDirIsHandledCorrectly) {
- llvm::StringMap<std::string> Index;
- Index["a"] = "/b/c/d";
- std::string IndexText = createCrossTUIndexString(Index);
-
- int IndexFD;
- llvm::SmallString<256> IndexFileName;
- ASSERT_FALSE(llvm::sys::fs::createTemporaryFile("index", "txt", IndexFD,
- IndexFileName));
- llvm::ToolOutputFile IndexFile(IndexFileName, IndexFD);
- IndexFile.os() << IndexText;
- IndexFile.os().flush();
- EXPECT_TRUE(llvm::sys::fs::exists(IndexFileName));
- llvm::Expected<llvm::StringMap<std::string>> IndexOrErr =
- parseCrossTUIndex(IndexFileName, "/ctudir");
- EXPECT_TRUE((bool)IndexOrErr);
- llvm::StringMap<std::string> ParsedIndex = IndexOrErr.get();
- EXPECT_EQ(ParsedIndex["a"], "/ctudir/b/c/d");
-}
-
} // end namespace cross_tu
} // end namespace clang
Index: clang/test/Analysis/ctu-unknown-parts-in-triples.cpp
===================================================================
--- clang/test/Analysis/ctu-unknown-parts-in-triples.cpp
+++ clang/test/Analysis/ctu-unknown-parts-in-triples.cpp
@@ -5,7 +5,7 @@
// RUN: mkdir -p %t/ctudir
// RUN: %clang_cc1 -std=c++14 -triple x86_64-pc-linux-gnu \
// RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp
-// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt
+// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
// RUN: %clang_analyze_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu \
// RUN: -analyzer-checker=core,debug.ExprInspection \
// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
Index: clang/test/Analysis/ctu-on-demand-parsing.cpp
===================================================================
--- /dev/null
+++ clang/test/Analysis/ctu-on-demand-parsing.cpp
@@ -0,0 +1,100 @@
+// RUN: rm -rf %t && mkdir %t
+// RUN: mkdir -p %t/ctudir
+// RUN: cp %S/Inputs/ctu-chain.cpp %t/ctudir/ctu-chain.cpp
+// RUN: echo '[{"directory":"%S/Inputs","command":"clang++ -c ctu-chain.cpp","file":"ctu-chain.cpp"},{"directory":"%S/Inputs","command":"clang++ -c ctu-other.cpp","file":"ctu-other.cpp"}]' | sed -e 's/\\/\\\\/g' > %t/ctudir/compile_commands.json
+// RUN: %clang_extdef_map %S/Inputs/ctu-chain.cpp %S/Inputs/ctu-other.cpp > %t/ctudir/externalDefMap.txt
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu \
+// RUN: -analyzer-checker=core,debug.ExprInspection \
+// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
+// RUN: -analyzer-config ctu-dir="%t/ctudir" \
+// RUN: -analyzer-config ctu-on-demand-parsing=true \
+// RUN: -analyzer-config ctu-on-demand-parsing-database="%t/ctudir/compile_commands.json" \
+// RUN: -verify %s
+// RUN: %clang_analyze_cc1 -triple x86_64-pc-linux-gnu \
+// RUN: -analyzer-checker=core,debug.ExprInspection \
+// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
+// RUN: -analyzer-config ctu-dir="%t/ctudir" \
+// RUN: -analyzer-config ctu-on-demand-parsing=true \
+// RUN: -analyzer-config ctu-on-demand-parsing-database="%t/ctudir/compile_commands.json" \
+// RUN: -analyzer-config display-ctu-progress=true 2>&1 %s | FileCheck %s
+
+// CHECK: CTU loaded AST file: {{.*}}ctu-other.cpp
+// CHECK: CTU loaded AST file: {{.*}}ctu-chain.cpp
+
+#include "ctu-hdr.h"
+
+void clang_analyzer_eval(int);
+
+int f(int);
+int g(int);
+int h(int);
+
+int callback_to_main(int x) { return x + 1; }
+
+namespace myns {
+int fns(int x);
+
+namespace embed_ns {
+int fens(int x);
+}
+
+class embed_cls {
+public:
+ int fecl(int x);
+};
+} // namespace myns
+
+class mycls {
+public:
+ int fcl(int x);
+ virtual int fvcl(int x);
+ static int fscl(int x);
+
+ class embed_cls2 {
+ public:
+ int fecl2(int x);
+ };
+};
+
+class derived : public mycls {
+public:
+ virtual int fvcl(int x) override;
+};
+
+namespace chns {
+int chf1(int x);
+}
+
+int fun_using_anon_struct(int);
+int other_macro_diag(int);
+
+void test_virtual_functions(mycls *obj) {
+ // The dynamic type is known.
+ clang_analyzer_eval(mycls().fvcl(1) == 8); // expected-warning{{TRUE}}
+ clang_analyzer_eval(derived().fvcl(1) == 9); // expected-warning{{TRUE}}
+ // We cannot decide about the dynamic type.
+ clang_analyzer_eval(obj->fvcl(1) == 8); // expected-warning{{FALSE}} expected-warning{{TRUE}}
+ clang_analyzer_eval(obj->fvcl(1) == 9); // expected-warning{{FALSE}} expected-warning{{TRUE}}
+}
+
+int main() {
+ clang_analyzer_eval(f(3) == 2); // expected-warning{{TRUE}}
+ clang_analyzer_eval(f(4) == 3); // expected-warning{{TRUE}}
+ clang_analyzer_eval(f(5) == 3); // expected-warning{{FALSE}}
+ clang_analyzer_eval(g(4) == 6); // expected-warning{{TRUE}}
+ clang_analyzer_eval(h(2) == 8); // expected-warning{{TRUE}}
+
+ clang_analyzer_eval(myns::fns(2) == 9); // expected-warning{{TRUE}}
+ clang_analyzer_eval(myns::embed_ns::fens(2) == -1); // expected-warning{{TRUE}}
+ clang_analyzer_eval(mycls().fcl(1) == 6); // expected-warning{{TRUE}}
+ clang_analyzer_eval(mycls::fscl(1) == 7); // expected-warning{{TRUE}}
+ clang_analyzer_eval(myns::embed_cls().fecl(1) == -6); // expected-warning{{TRUE}}
+ clang_analyzer_eval(mycls::embed_cls2().fecl2(0) == -11); // expected-warning{{TRUE}}
+
+ clang_analyzer_eval(chns::chf1(4) == 12); // expected-warning{{TRUE}}
+ clang_analyzer_eval(fun_using_anon_struct(8) == 8); // expected-warning{{TRUE}}
+
+ clang_analyzer_eval(other_macro_diag(1) == 1); // expected-warning{{TRUE}}
+ // expected-warning@Inputs/ctu-other.cpp:93{{REACHABLE}}
+ MACRODIAG(); // expected-warning{{REACHABLE}}
+}
Index: clang/test/Analysis/ctu-on-demand-parsing.c
===================================================================
--- /dev/null
+++ clang/test/Analysis/ctu-on-demand-parsing.c
@@ -0,0 +1,71 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: cp "%s" "%t/ctu-on-demand-parsing.c"
+// RUN: cp "%S/Inputs/ctu-other.c" "%t/ctu-other.c"
+// RUN: echo '[{"directory":"%t","command":"gcc -c -std=c89 -Wno-visibility ctu-other.c","file":"ctu-other.c"}]' | sed -e 's/\\/\\\\/g' > %t/compile_commands.json
+// RUN: cd "%t" && %clang_extdef_map ctu-other.c > externalDefMap.txt
+// RUN: cd "%t" && %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -std=c89 -analyze \
+// RUN: -analyzer-checker=core,debug.ExprInspection \
+// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
+// RUN: -analyzer-config ctu-dir=. \
+// RUN: -analyzer-config ctu-on-demand-parsing=true \
+// RUN: -verify ctu-on-demand-parsing.c
+
+void clang_analyzer_eval(int);
+
+// Test typedef and global variable in function.
+typedef struct {
+ int a;
+ int b;
+} FooBar;
+extern FooBar fb;
+int f(int);
+void testGlobalVariable() {
+ clang_analyzer_eval(f(5) == 1); // expected-warning{{TRUE}}
+}
+
+// Test enums.
+int enumCheck(void);
+enum A { x, y, z };
+void testEnum() {
+ clang_analyzer_eval(x == 0); // expected-warning{{TRUE}}
+ clang_analyzer_eval(enumCheck() == 42); // expected-warning{{TRUE}}
+}
+
+// Test that asm import does not fail.
+int inlineAsm();
+int testInlineAsm() { return inlineAsm(); }
+
+// Test reporting error in a macro.
+struct S;
+int g(struct S *);
+void testMacro(void) {
+ g(0);
+ // expected-warning@ctu-other.c:29 {{Access to field 'a' results in a dereference of a null pointer (loaded from variable 'ctx')}}
+}
+
+// The external function prototype is incomplete.
+// warning:implicit functions are prohibited by c99
+void testImplicit() {
+ int res = identImplicit(6); // external implicit functions are not inlined
+ clang_analyzer_eval(res == 6); // expected-warning{{TRUE}}
+ // Call something with uninitialized from the same function in which the
+ // implicit was called. This is necessary to reproduce a special bug in
+ // NoStoreFuncVisitor.
+ int uninitialized;
+ h(uninitialized); // expected-warning{{1st function call argument is an uninitialized value}}
+}
+
+// Tests the import of functions that have a struct parameter
+// defined in its prototype.
+struct DataType {
+ int a;
+ int b;
+};
+int structInProto(struct DataType *d);
+void testStructDefInArgument() {
+ struct DataType d;
+ d.a = 1;
+ d.b = 0;
+ clang_analyzer_eval(structInProto(&d) == 0); // expected-warning{{TRUE}} expected-warning{{FALSE}}
+}
Index: clang/test/Analysis/ctu-main.cpp
===================================================================
--- clang/test/Analysis/ctu-main.cpp
+++ clang/test/Analysis/ctu-main.cpp
@@ -4,7 +4,7 @@
// RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp
// RUN: %clang_cc1 -std=c++14 -triple x86_64-pc-linux-gnu \
// RUN: -emit-pch -o %t/ctudir/ctu-chain.cpp.ast %S/Inputs/ctu-chain.cpp
-// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt
+// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
// RUN: %clang_analyze_cc1 -std=c++14 -triple x86_64-pc-linux-gnu \
// RUN: -analyzer-checker=core,debug.ExprInspection \
// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
Index: clang/test/Analysis/ctu-main.c
===================================================================
--- clang/test/Analysis/ctu-main.c
+++ clang/test/Analysis/ctu-main.c
@@ -2,7 +2,7 @@
// RUN: mkdir -p %t/ctudir2
// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu \
// RUN: -emit-pch -o %t/ctudir2/ctu-other.c.ast %S/Inputs/ctu-other.c
-// RUN: cp %S/Inputs/ctu-other.c.externalDefMap.txt %t/ctudir2/externalDefMap.txt
+// RUN: cp %S/Inputs/ctu-other.c.externalDefMap.ast-dump.txt %t/ctudir2/externalDefMap.txt
// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -std=c89 -analyze \
// RUN: -analyzer-checker=core,debug.ExprInspection \
// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
@@ -50,6 +50,10 @@
void testImplicit() {
int res = identImplicit(6); // external implicit functions are not inlined
clang_analyzer_eval(res == 6); // expected-warning{{TRUE}}
+ // Call something with uninitialized from the same function in which the implicit was called.
+ // This is necessary to reproduce a special bug in NoStoreFuncVisitor.
+ int uninitialized;
+ h(uninitialized); // expected-warning{{1st function call argument is an uninitialized value}}
}
// Tests the import of functions that have a struct parameter
Index: clang/test/Analysis/ctu-different-triples.cpp
===================================================================
--- clang/test/Analysis/ctu-different-triples.cpp
+++ clang/test/Analysis/ctu-different-triples.cpp
@@ -2,7 +2,7 @@
// RUN: mkdir -p %t/ctudir
// RUN: %clang_cc1 -std=c++14 -triple x86_64-pc-linux-gnu \
// RUN: -emit-pch -o %t/ctudir/ctu-other.cpp.ast %S/Inputs/ctu-other.cpp
-// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.txt %t/ctudir/externalDefMap.txt
+// RUN: cp %S/Inputs/ctu-other.cpp.externalDefMap.ast-dump.txt %t/ctudir/externalDefMap.txt
// RUN: %clang_analyze_cc1 -std=c++14 -triple powerpc64-montavista-linux-gnu \
// RUN: -analyzer-checker=core,debug.ExprInspection \
// RUN: -analyzer-config experimental-enable-naive-ctu-analysis=true \
Index: clang/test/Analysis/analyzer-config.c
===================================================================
--- clang/test/Analysis/analyzer-config.c
+++ clang/test/Analysis/analyzer-config.c
@@ -33,6 +33,8 @@
// CHECK-NEXT: ctu-dir = ""
// CHECK-NEXT: ctu-import-threshold = 100
// CHECK-NEXT: ctu-index-name = externalDefMap.txt
+// CHECK-NEXT: ctu-on-demand-parsing = false
+// CHECK-NEXT: ctu-on-demand-parsing-database = compile_commands.json
// CHECK-NEXT: deadcode.DeadStores:ShowFixIts = false
// CHECK-NEXT: deadcode.DeadStores:WarnForDeadNestedAssignments = true
// CHECK-NEXT: debug.AnalysisOrder:* = false
@@ -100,4 +102,4 @@
// CHECK-NEXT: unroll-loops = false
// CHECK-NEXT: widen-loops = false
// CHECK-NEXT: [stats]
-// CHECK-NEXT: num-entries = 97
+// CHECK-NEXT: num-entries = 99
Index: clang/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.txt
===================================================================
--- /dev/null
+++ clang/test/Analysis/Inputs/ctu-other.cpp.externalDefMap.txt
@@ -1,30 +0,0 @@
-c:@N@chns@F@chf1#I# ctu-other.cpp.ast
-c:@N@myns@N@embed_ns@F@fens#I# ctu-other.cpp.ast
-c:@F@g#I# ctu-other.cpp.ast
-c:@S@mycls@F@fscl#I#S ctu-other.cpp.ast
-c:@S@mycls@F@fcl#I# ctu-other.cpp.ast
-c:@S@mycls@F@fvcl#I# ctu-other.cpp.ast
-c:@N@myns@S@embed_cls@F@fecl#I# ctu-other.cpp.ast
-c:@S@mycls@S@embed_cls2@F@fecl2#I# ctu-other.cpp.ast
-c:@S@derived@F@fvcl#I# ctu-other.cpp.ast
-c:@F@f#I# ctu-other.cpp.ast
-c:@N@myns@F@fns#I# ctu-other.cpp.ast
-c:@F@h#I# ctu-other.cpp.ast
-c:@F@h_chain#I# ctu-chain.cpp.ast
-c:@N@chns@S@chcls@F@chf4#I# ctu-chain.cpp.ast
-c:@N@chns@F@chf2#I# ctu-chain.cpp.ast
-c:@F@fun_using_anon_struct#I# ctu-other.cpp.ast
-c:@F@other_macro_diag#I# ctu-other.cpp.ast
-c:@extInt ctu-other.cpp.ast
-c:@N@intns@extInt ctu-other.cpp.ast
-c:@extS ctu-other.cpp.ast
-c:@S@A@a ctu-other.cpp.ast
-c:@extSC ctu-other.cpp.ast
-c:@S@ST@sc ctu-other.cpp.ast
-c:@extSCN ctu-other.cpp.ast
-c:@extSubSCN ctu-other.cpp.ast
-c:@extSCC ctu-other.cpp.ast
-c:@extU ctu-other.cpp.ast
-c:@S@TestAnonUnionUSR@Test ctu-other.cpp.ast
-c:@F@testImportOfIncompleteDefaultParmDuringImport#I# ctu-other.cpp.ast
-c:@F@testImportOfDelegateConstructor#I# ctu-other.cpp.ast
\ No newline at end of file
Index: clang/test/Analysis/Inputs/ctu-other.c.externalDefMap.txt
===================================================================
--- /dev/null
+++ clang/test/Analysis/Inputs/ctu-other.c.externalDefMap.txt
@@ -1,6 +0,0 @@
-c:@F@inlineAsm ctu-other.c.ast
-c:@F@g ctu-other.c.ast
-c:@F@f ctu-other.c.ast
-c:@F@enumCheck ctu-other.c.ast
-c:@F@identImplicit ctu-other.c.ast
-c:@F@structInProto ctu-other.c.ast
Index: clang/test/Analysis/Inputs/ctu-other.c
===================================================================
--- clang/test/Analysis/Inputs/ctu-other.c
+++ clang/test/Analysis/Inputs/ctu-other.c
@@ -31,9 +31,11 @@
}
// Test that asm import does not fail.
+// TODO: Support the GNU extension asm keyword as well.
+// Example using the GNU extension: asm("mov $42, %0" : "=r"(res));
int inlineAsm() {
int res;
- asm("mov $42, %0"
+ __asm__("mov $42, %0"
: "=r"(res));
return res;
}
Index: clang/lib/StaticAnalyzer/Core/CallEvent.cpp
===================================================================
--- clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -574,6 +574,11 @@
cross_tu::CrossTranslationUnitContext &CTUCtx =
*Engine.getCrossTranslationUnitContext();
+
+ // Optional<StringRef> OnDemandParsingDatabase;
+ // if (Opts.CTUOnDemandParsing)
+ // OnDemandParsingDatabase = Opts.CTUOnDemandParsingDatabase;
+
llvm::Expected<const FunctionDecl *> CTUDeclOrError =
CTUCtx.getCrossTUDefinition(FD, Opts.CTUDir, Opts.CTUIndexName,
Opts.DisplayCTUProgress);
Index: clang/lib/Frontend/CompilerInvocation.cpp
===================================================================
--- clang/lib/Frontend/CompilerInvocation.cpp
+++ clang/lib/Frontend/CompilerInvocation.cpp
@@ -510,6 +510,12 @@
Diags->Report(diag::err_analyzer_config_invalid_input) << "ctu-dir"
<< "a filename";
+ if (AnOpts.CTUOnDemandParsing &&
+ !llvm::sys::fs::exists(AnOpts.CTUOnDemandParsingDatabase))
+ Diags->Report(diag::err_analyzer_config_invalid_input)
+ << "ctu-on-demand-parsing-database"
+ << "a filename";
+
if (!AnOpts.ModelPath.empty() &&
!llvm::sys::fs::is_directory(AnOpts.ModelPath))
Diags->Report(diag::err_analyzer_config_invalid_input) << "model-path"
Index: clang/lib/CrossTU/CrossTranslationUnit.cpp
===================================================================
--- clang/lib/CrossTU/CrossTranslationUnit.cpp
+++ clang/lib/CrossTU/CrossTranslationUnit.cpp
@@ -18,12 +18,16 @@
#include "clang/Frontend/CompilerInstance.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Index/USRGeneration.h"
-#include "llvm/ADT/Triple.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/Optional.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Triple.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
#include <fstream>
#include <sstream>
@@ -100,6 +104,8 @@
return "Failed to import the definition.";
case index_error_code::failed_to_get_external_ast:
return "Failed to load external AST source.";
+ case index_error_code::failed_to_load_compilation_database:
+ return "Failed to load compilation database.";
case index_error_code::failed_to_generate_usr:
return "Failed to generate USR.";
case index_error_code::triple_mismatch:
@@ -110,6 +116,9 @@
return "Language dialect mismatch";
case index_error_code::load_threshold_reached:
return "Load threshold reached";
+ case index_error_code::ambiguous_compile_commands_database:
+ return "Compile commands database contains multiple references to the "
+ "same sorce file.";
}
llvm_unreachable("Unrecognized index_error_code.");
}
@@ -129,7 +138,7 @@
}
llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir) {
+parseCrossTUIndex(StringRef IndexPath) {
std::ifstream ExternalMapFile{std::string(IndexPath)};
if (!ExternalMapFile)
return llvm::make_error<IndexError>(index_error_code::missing_index_file,
@@ -147,9 +156,7 @@
return llvm::make_error<IndexError>(
index_error_code::multiple_definitions, IndexPath.str(), LineNo);
StringRef FileName = LineRef.substr(Pos + 1);
- SmallString<256> FilePath = CrossTUDir;
- llvm::sys::path::append(FilePath, FileName);
- Result[LookupName] = std::string(FilePath);
+ Result[LookupName] = FileName.str();
} else
return llvm::make_error<IndexError>(
index_error_code::invalid_index_format, IndexPath.str(), LineNo);
@@ -341,12 +348,12 @@
}
}
-CrossTranslationUnitContext::ASTFileLoader::ASTFileLoader(
- const CompilerInstance &CI)
- : CI(CI) {}
+CrossTranslationUnitContext::ASTFileLoader::ASTFileLoader(CompilerInstance &CI,
+ StringRef CTUDir)
+ : CI(CI), CTUDir(CTUDir) {}
-std::unique_ptr<ASTUnit>
-CrossTranslationUnitContext::ASTFileLoader::operator()(StringRef ASTFilePath) {
+CrossTranslationUnitContext::LoadResultTy
+CrossTranslationUnitContext::ASTFileLoader::load(StringRef Identifier) {
// Load AST from ast-dump.
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
TextDiagnosticPrinter *DiagClient =
@@ -355,16 +362,121 @@
IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient));
+ llvm::SmallString<256> AbsPath = CTUDir;
+ llvm::sys::path::append(AbsPath, Identifier);
+
return ASTUnit::LoadFromASTFile(
- std::string(ASTFilePath), CI.getPCHContainerOperations()->getRawReader(),
- ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts());
+ std::string(AbsPath.str()),
+ CI.getPCHContainerOperations()->getRawReader(), ASTUnit::LoadEverything,
+ Diags, CI.getFileSystemOpts());
+}
+
+/// Load the AST from a source-file, which is supposed to be located inside the
+/// compilation database \p OnDemandParsingCommands. The compilation database
+/// can contain the path of the file under the key "file" as an absolute path,
+/// or as a relative path. When emitting diagnostics, plist files may contain
+/// references to a location in a TU, that is different from the main TU. In
+/// such cases, the file path emitted by the DiagnosticEngine is based on how
+/// the exact invocation is assembled inside the ClangTool, which performs the
+/// building of the ASTs. In order to ensure absolute paths inside the
+/// diagnostics, we use the ArgumentsAdjuster API of ClangTool to make sure that
+/// the invocation inside ClangTool is always made with an absolute path. \p
+/// ASTSourcePath is assumed to be the lookup-name of the file, which comes from
+/// the Index. The Index is built by the \p clang-extdef-mapping tool, which is
+/// supposed to generate absolute paths.
+///
+/// We must have absolute paths inside the plist, because otherwise we would
+/// not be able to parse the bug, because we could not find the files with
+/// relative paths. The directory of one entry in the compilation db may be
+/// different from the directory where the plist is interpreted.
+///
+/// Note that as the ClangTool is instantiated with a lookup-vector, which
+/// contains a single entry; the supposedly absolute path of the source file.
+/// So, the ArgumentAdjuster will only be used on the single corresponding
+/// invocation. This guarantees that even if two files match in name, but
+/// differ in location, only the correct one's invocation will be handled. This
+/// is due to the fact that the lookup is done correctly inside the
+/// OnDemandParsingDatabase, so it works for already absolute paths given under
+/// the "file" entry of the compilation database, but also if a relative path is
+/// given. In such a case, the lookup uses the "directory" entry as well to
+/// identify the correct file.
+CrossTranslationUnitContext::LoadResultTy
+CrossTranslationUnitContext::ASTOnDemandLoader::load(StringRef Identifier) {
+
+ if (auto InitError = lazyInitCompileCommands())
+ return std::move(InitError);
+
+ using namespace tooling;
+
+ SmallVector<std::string, 1> Files;
+ Files.push_back(std::string(Identifier));
+ ClangTool Tool(*CompileCommands, Files, CI.getPCHContainerOperations());
+
+ /// Lambda filter designed to find the source file argument inside an
+ /// invocation used to build the ASTs, and replace it with its absolute path
+ /// equivalent.
+ auto SourcePathNormalizer = [Identifier](const CommandLineArguments &Args,
+ StringRef FileName) {
+ /// Match the argument to the absolute path by checking whether it is a
+ /// postfix.
+ auto IsPostfixOfLookup = [Identifier](const std::string &Arg) {
+ return Identifier.rfind(Arg) != llvm::StringRef::npos;
+ };
+
+ /// Commandline arguments are modified, and the API dictates the return of
+ /// a new instance, so copy the original.
+ CommandLineArguments Result{Args};
+
+ /// Search for the source file argument. Start from the end as a heuristic,
+ /// as most invocations tend to contain the source file argument in their
+ /// latter half. Only the first match is replaced.
+ auto SourceFilePath =
+ std::find_if(Result.rbegin(), Result.rend(), IsPostfixOfLookup);
+
+ /// If source file argument could not been found, return the original
+ /// CommandlineArgumentsInstance.
+ if (SourceFilePath == Result.rend())
+ return Result;
+
+ /// Overwrite the argument with the \p ASTSourcePath, as it is assumed to
+ /// be the absolute path of the file.
+ *SourceFilePath = Identifier.str();
+
+ return Result;
+ };
+
+ Tool.appendArgumentsAdjuster(std::move(SourcePathNormalizer));
+
+ std::vector<std::unique_ptr<ASTUnit>> ASTs;
+ Tool.buildASTs(ASTs);
+
+ /// There is an assumption that the compilation database does not contain
+ /// multiple entries for the same source file.
+ if (ASTs.size() > 1)
+ return llvm::make_error<IndexError>(
+ index_error_code::ambiguous_compile_commands_database);
+
+ /// Ideally there is exactly one entry in the compilation database that
+ /// matchse the source file.
+ if (ASTs.size() != 1)
+ return llvm::make_error<IndexError>(
+ index_error_code::failed_to_get_external_ast);
+
+ ASTs[0]->enableSourceFileDiagnostics();
+ return std::move(ASTs[0]);
}
CrossTranslationUnitContext::ASTUnitStorage::ASTUnitStorage(
- const CompilerInstance &CI)
- : FileAccessor(CI), LoadGuard(const_cast<CompilerInstance &>(CI)
- .getAnalyzerOpts()
- ->CTUImportThreshold) {}
+ CompilerInstance &CI)
+ : LoadGuard(CI.getAnalyzerOpts()->CTUImportThreshold) {
+
+ AnalyzerOptionsRef Opts = CI.getAnalyzerOpts();
+ if (Opts->CTUOnDemandParsing)
+ Loader = std::make_unique<ASTOnDemandLoader>(
+ CI, Opts->CTUOnDemandParsingDatabase);
+ else
+ Loader = std::make_unique<ASTFileLoader>(CI, Opts->CTUDir);
+}
llvm::Expected<ASTUnit *>
CrossTranslationUnitContext::ASTUnitStorage::getASTUnitForFile(
@@ -380,8 +492,12 @@
index_error_code::load_threshold_reached);
}
- // Load the ASTUnit from the pre-dumped AST file specified by ASTFileName.
- std::unique_ptr<ASTUnit> LoadedUnit = FileAccessor(FileName);
+ auto LoadAttempt = Loader->load(FileName);
+
+ if (!LoadAttempt)
+ return LoadAttempt.takeError();
+
+ std::unique_ptr<ASTUnit> LoadedUnit = std::move(LoadAttempt.get());
// Need the raw pointer and the unique_ptr as well.
ASTUnit *Unit = LoadedUnit.get();
@@ -461,7 +577,7 @@
else
llvm::sys::path::append(IndexFile, IndexName);
- if (auto IndexMapping = parseCrossTUIndex(IndexFile, CrossTUDir)) {
+ if (auto IndexMapping = parseCrossTUIndex(IndexFile)) {
// Initialize member map.
NameFileMap = *IndexMapping;
return llvm::Error::success();
@@ -471,6 +587,26 @@
};
}
+llvm::Error
+CrossTranslationUnitContext::ASTOnDemandLoader::lazyInitCompileCommands() {
+ // Lazily initialize the compilation database.
+
+ if (CompileCommands)
+ return llvm::Error::success();
+
+ std::string LoadError;
+ CompileCommands = tooling::JSONCompilationDatabase::loadFromFile(
+ OnDemandParsingDatabase, LoadError,
+ tooling::JSONCommandLineSyntax::AutoDetect);
+ return CompileCommands ? llvm::Error::success()
+ : llvm::make_error<IndexError>(
+ index_error_code::failed_to_get_external_ast);
+}
+
+CrossTranslationUnitContext::ASTOnDemandLoader::ASTOnDemandLoader(
+ CompilerInstance &CI, StringRef OnDemandParsingDatabase)
+ : CI(CI), OnDemandParsingDatabase(OnDemandParsingDatabase) {}
+
llvm::Expected<ASTUnit *> CrossTranslationUnitContext::loadExternalAST(
StringRef LookupName, StringRef CrossTUDir, StringRef IndexName,
bool DisplayCTUProgress) {
Index: clang/lib/CrossTU/CMakeLists.txt
===================================================================
--- clang/lib/CrossTU/CMakeLists.txt
+++ clang/lib/CrossTU/CMakeLists.txt
@@ -10,4 +10,5 @@
clangBasic
clangFrontend
clangIndex
+ clangTooling
)
Index: clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
===================================================================
--- clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
+++ clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.def
@@ -377,6 +377,21 @@
"the name of the file containing the CTU index of definitions.",
"externalDefMap.txt")
+ANALYZER_OPTION(bool, CTUOnDemandParsing, "ctu-on-demand-parsing",
+ "Whether to parse function definitions from external TUs in "
+ "an on-demand manner during analysis. When using on-demand "
+ "parsing there is no need for pre-dumping ASTs. External "
+ "definition mapping is still needed, and a valid compilation "
+ "database with compile commands for the external TUs is also "
+ "necessary. Disabled by default.",
+ false)
+
+ANALYZER_OPTION(StringRef, CTUOnDemandParsingDatabase,
+ "ctu-on-demand-parsing-database",
+ "The path to the compilation database used for on-demand "
+ "parsing of ASTs during CTU analysis.",
+ "compile_commands.json")
+
ANALYZER_OPTION(
StringRef, ModelPath, "model-path",
"The analyzer can inline an alternative implementation written in C at the "
Index: clang/include/clang/CrossTU/CrossTranslationUnit.h
===================================================================
--- clang/include/clang/CrossTU/CrossTranslationUnit.h
+++ clang/include/clang/CrossTU/CrossTranslationUnit.h
@@ -20,6 +20,8 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/StringMap.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/IR/OperandTraits.h"
#include "llvm/Support/Error.h"
namespace clang {
@@ -33,6 +35,10 @@
class NamedDecl;
class TranslationUnitDecl;
+namespace tooling {
+class JSONCompilationDatabase;
+}
+
namespace cross_tu {
enum class index_error_code {
@@ -42,12 +48,14 @@
multiple_definitions,
missing_definition,
failed_import,
+ failed_to_load_compilation_database,
failed_to_get_external_ast,
failed_to_generate_usr,
triple_mismatch,
lang_mismatch,
lang_dialect_mismatch,
- load_threshold_reached
+ load_threshold_reached,
+ ambiguous_compile_commands_database
};
class IndexError : public llvm::ErrorInfo<IndexError> {
@@ -86,7 +94,7 @@
/// \return Returns a map where the USR is the key and the filepath is the value
/// or an error.
llvm::Expected<llvm::StringMap<std::string>>
-parseCrossTUIndex(StringRef IndexPath, StringRef CrossTUDir);
+parseCrossTUIndex(StringRef IndexPath);
std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index);
@@ -209,14 +217,45 @@
/// imported the FileID.
ImportedFileIDMap ImportedFileIDs;
- /// Functor for loading ASTUnits from AST-dump files.
- class ASTFileLoader {
+ using LoadResultTy = llvm::Expected<std::unique_ptr<ASTUnit>>;
+
+ struct ASTLoader {
+ /// Load the ASTUnit by an identifier. Subclasses should determine what this
+ /// would be.
+ virtual LoadResultTy load(StringRef Identifier) = 0;
+ virtual ~ASTLoader() = default;
+ };
+
+ /// Implementation for loading ASTUnits from AST-dump files.
+ class ASTFileLoader : public ASTLoader {
public:
- ASTFileLoader(const CompilerInstance &CI);
- std::unique_ptr<ASTUnit> operator()(StringRef ASTFilePath);
+ explicit ASTFileLoader(CompilerInstance &CI, StringRef CTUDir);
+
+ /// ASTFileLoader uses a the path of the dump file as Identifier.
+ LoadResultTy load(StringRef Identifier) override;
private:
- const CompilerInstance &CI;
+ CompilerInstance &CI;
+ StringRef CTUDir;
+ };
+
+ /// Implementation for loading ASTUnits by parsing them on-demand.
+ class ASTOnDemandLoader : public ASTLoader {
+ public:
+ ASTOnDemandLoader(CompilerInstance &CI, StringRef OnDemandParsingDatabase);
+
+ /// ASTOnDemandLoader uses the path of the source file to be parsed as
+ /// Identifier.
+ LoadResultTy load(StringRef Identifier) override;
+
+ llvm::Error lazyInitCompileCommands();
+
+ private:
+ CompilerInstance &CI;
+ StringRef OnDemandParsingDatabase;
+ /// In case of on-demand parsing, the compilation database is parsed and
+ /// stored.
+ std::unique_ptr<tooling::JSONCompilationDatabase> CompileCommands;
};
/// Maintain number of AST loads and check for reaching the load limit.
@@ -242,7 +281,7 @@
/// are the concerns of ASTUnitStorage class.
class ASTUnitStorage {
public:
- ASTUnitStorage(const CompilerInstance &CI);
+ ASTUnitStorage(CompilerInstance &CI);
/// Loads an ASTUnit for a function.
///
/// \param FunctionName USR name of the function.
@@ -287,18 +326,16 @@
using IndexMapTy = BaseMapTy<std::string>;
IndexMapTy NameFileMap;
- ASTFileLoader FileAccessor;
+ std::unique_ptr<ASTLoader> Loader;
- /// Limit the number of loaded ASTs. Used to limit the memory usage of the
- /// CrossTranslationUnitContext.
- /// The ASTUnitStorage has the knowledge about if the AST to load is
- /// actually loaded or returned from cache. This information is needed to
- /// maintain the counter.
+ /// Limit the number of loaded ASTs. It is used to limit the memory usage
+ /// of the CrossTranslationUnitContext. The ASTUnitStorage has the
+ /// information whether the AST to load is actually loaded or returned from
+ /// cache. This information is needed to maintain the counter.
ASTLoadGuard LoadGuard;
};
ASTUnitStorage ASTStorage;
-
};
} // namespace cross_tu
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
