boris updated this revision to Diff 116438.
boris marked an inline comment as done.
boris added a comment.
New revision this time with the tests (which got dropped from the earlier
revision diff for some reason).
https://reviews.llvm.org/D37299
Files:
docs/Modules.rst
include/clang/Basic/DiagnosticDriverKinds.td
include/clang/Driver/Options.td
lib/Driver/ToolChains/Clang.cpp
lib/Frontend/CompilerInvocation.cpp
test/CXX/modules-ts/basic/basic.search/module-import.cpp
Index: test/CXX/modules-ts/basic/basic.search/module-import.cpp
===================================================================
--- test/CXX/modules-ts/basic/basic.search/module-import.cpp
+++ test/CXX/modules-ts/basic/basic.search/module-import.cpp
@@ -5,6 +5,8 @@
// RUN: echo 'export module x; int a, b;' > %t/x.cppm
// RUN: echo 'export module y; import x; int c;' > %t/y.cppm
// RUN: echo 'export module z; import y; int d;' > %t/z.cppm
+// RUN: echo 'x=%t/x.pcm' > %t/modmap
+// RUN: echo 'y=%t/y.pcm' >> %t/modmap
//
// RUN: %clang_cc1 -std=c++1z -fmodules-ts -emit-module-interface %t/x.cppm -o %t/x.pcm
// RUN: %clang_cc1 -std=c++1z -fmodules-ts -emit-module-interface -fmodule-file=%t/x.pcm %t/y.cppm -o %t/y.pcm
@@ -19,7 +21,17 @@
// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file=y=%t/y.pcm -verify %s \
// RUN: -DMODULE_NAME=y
//
+// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file-map=%t/modmap -verify %s \
+// RUN: -DMODULE_NAME=x
+// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file-map=%t/modmap -verify %s \
+// RUN: -DMODULE_NAME=y
+//
// RUN: mv %t/x.pcm %t/a.pcm
+// RUN: echo 'foo.o: foo.cxx' > %t/modmap
+// RUN: echo '# Mmodule name to file mapping:' >> %t/modmap
+// RUN: echo '#@z=%t/z.pcm' >> %t/modmap
+// RUN: echo '#@ y=%t/y.pcm' >> %t/modmap
+// RUN: echo '#@x=%t/a.pcm ' >> %t/modmap
//
// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file=x=%t/a.pcm -verify %s \
// RUN: -DMODULE_NAME=x
@@ -33,7 +45,15 @@
// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file=z=%t/z.pcm -fmodule-file=y=%t/y.pcm -fmodule-file=x=%t/a.pcm -verify %s \
// RUN: -DMODULE_NAME=z
//
+//
+// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file-map=#@=%t/modmap -verify %s \
+// RUN: -DMODULE_NAME=x
+// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file-map=#@=%t/modmap -verify %s \
+// RUN: -DMODULE_NAME=y
+// RUN: %clang_cc1 -std=c++1z -fmodules-ts -I%t -fmodule-file-map=#@=%t/modmap -verify %s \
+// RUN: -DMODULE_NAME=z
+//
import MODULE_NAME;
// expected-no-diagnostics
Index: lib/Frontend/CompilerInvocation.cpp
===================================================================
--- lib/Frontend/CompilerInvocation.cpp
+++ lib/Frontend/CompilerInvocation.cpp
@@ -1528,8 +1528,80 @@
return P.str();
}
+// Read the mapping of module names to precompiled module files from a file.
+// The argument can include an optional line prefix ([<prefix>]=<file>), in
+// which case only lines that start with the prefix are considered (with the
+// prefix and the following whitespaces, if any, ignored).
+//
+// Each mapping entry should be in the same form as the -fmodule-file option
+// value (<name>=<file>) with leading/trailing whitespaces ignored.
+//
+static void LoadModuleFileMap(HeaderSearchOptions &Opts,
+ DiagnosticsEngine &Diags, FileManager &FileMgr,
+ StringRef Val, const std::string &Arg) {
+ // See if we have the prefix.
+ StringRef File;
+ StringRef Prefix;
+ if (Val.find('=') != StringRef::npos) {
+ auto Pair = Val.split('=');
+ Prefix = Pair.first;
+ File = Pair.second;
+ if (Prefix.empty()) {
+ Diags.Report(diag::err_drv_invalid_value) << Arg << Val;
+ return;
+ }
+ } else
+ File = Val;
+
+ if (File.empty()) {
+ Diags.Report(diag::err_drv_invalid_value) << Arg << Val;
+ return;
+ }
+
+ auto *Buf = FileMgr.getBufferForFile(File);
+ if (!Buf) {
+ Diags.Report(diag::err_cannot_open_file)
+ << File << Buf.getError().message();
+ return;
+ }
+
+ // Read the file line by line.
+ StringRef Str = Buf.get()->getBuffer();
+ for (size_t B = 0, E = 0; B < Str.size(); B = E + 1) {
+ E = Str.find_first_of(StringRef("\n\0", 2), B);
+
+ if (E == StringRef::npos)
+ E = Str.size();
+ else if (Str[E] == '\0')
+ break; // The file (or the rest of it) is binary, bail out.
+
+ // [B, E) is our line. Compare and skip the prefix, if any.
+ StringRef Line = Str.substr(B, E - B);
+ if (!Prefix.empty()) {
+ if (!Line.startswith(Prefix))
+ continue;
+
+ Line = Line.substr(Prefix.size());
+ }
+
+ // Skip leading and trailing whitespaces and ignore blanks (even if they
+ // had prefix; think make comments).
+ Line = Line.trim();
+ if (Line.empty())
+ continue;
+
+ if (Line.find('=') == StringRef::npos) {
+ Diags.Report(diag::err_drv_invalid_module_file_map) << Line;
+ continue;
+ }
+
+ Opts.PrebuiltModuleFiles.insert(Line.split('='));
+ }
+}
+
static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
- const std::string &WorkingDir) {
+ DiagnosticsEngine &Diags,
+ FileManager &FileMgr) {
using namespace options;
Opts.Sysroot = Args.getLastArgValue(OPT_isysroot, "/");
Opts.Verbose = Args.hasArg(OPT_v);
@@ -1543,21 +1615,24 @@
// Canonicalize -fmodules-cache-path before storing it.
SmallString<128> P(Args.getLastArgValue(OPT_fmodules_cache_path));
if (!(P.empty() || llvm::sys::path::is_absolute(P))) {
+ const std::string &WorkingDir (FileMgr.getFileSystemOpts().WorkingDir);
if (WorkingDir.empty())
llvm::sys::fs::make_absolute(P);
else
llvm::sys::fs::make_absolute(WorkingDir, P);
}
llvm::sys::path::remove_dots(P);
Opts.ModuleCachePath = P.str();
Opts.ModuleUserBuildPath = Args.getLastArgValue(OPT_fmodules_user_build_path);
// Only the -fmodule-file=<name>=<file> form.
for (const Arg *A : Args.filtered(OPT_fmodule_file)) {
StringRef Val = A->getValue();
if (Val.find('=') != StringRef::npos)
Opts.PrebuiltModuleFiles.insert(Val.split('='));
}
+ for (const Arg *A : Args.filtered(OPT_fmodule_file_map))
+ LoadModuleFileMap(Opts, Diags, FileMgr, A->getValue(), A->getAsString(Args));
for (const Arg *A : Args.filtered(OPT_fprebuilt_module_path))
Opts.AddPrebuiltModulePath(A->getValue());
Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
@@ -2511,7 +2586,6 @@
}
static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
- FileManager &FileMgr,
DiagnosticsEngine &Diags,
frontend::ActionKind Action) {
using namespace options;
@@ -2680,14 +2754,19 @@
false /*DefaultDiagColor*/, false /*DefaultShowOpt*/);
ParseCommentArgs(LangOpts.CommentOpts, Args);
ParseFileSystemArgs(Res.getFileSystemOpts(), Args);
+
+ // File manager used during option parsing (e.g., for loading map files,
+ // etc).
+ //
+ FileManager FileMgr(Res.getFileSystemOpts());
+
// FIXME: We shouldn't have to pass the DashX option around here
InputKind DashX = ParseFrontendArgs(Res.getFrontendOpts(), Args, Diags,
LangOpts.IsHeaderFile);
ParseTargetArgs(Res.getTargetOpts(), Args, Diags);
Success &= ParseCodeGenArgs(Res.getCodeGenOpts(), Args, DashX, Diags,
Res.getTargetOpts());
- ParseHeaderSearchArgs(Res.getHeaderSearchOpts(), Args,
- Res.getFileSystemOpts().WorkingDir);
+ ParseHeaderSearchArgs(Res.getHeaderSearchOpts(), Args, Diags, FileMgr);
if (DashX.getFormat() == InputKind::Precompiled ||
DashX.getLanguage() == InputKind::LLVM_IR) {
// ObjCAAutoRefCount and Sanitize LangOpts are used to setup the
@@ -2728,60 +2807,55 @@
!LangOpts.Sanitize.has(SanitizerKind::Address) &&
!LangOpts.Sanitize.has(SanitizerKind::Memory);
- // FIXME: ParsePreprocessorArgs uses the FileManager to read the contents of
- // PCH file and find the original header name. Remove the need to do that in
- // ParsePreprocessorArgs and remove the FileManager
- // parameters from the function and the "FileManager.h" #include.
- FileManager FileMgr(Res.getFileSystemOpts());
- ParsePreprocessorArgs(Res.getPreprocessorOpts(), Args, FileMgr, Diags,
+ ParsePreprocessorArgs(Res.getPreprocessorOpts(), Args, Diags,
Res.getFrontendOpts().ProgramAction);
ParsePreprocessorOutputArgs(Res.getPreprocessorOutputOpts(), Args,
Res.getFrontendOpts().ProgramAction);
// Turn on -Wspir-compat for SPIR target.
llvm::Triple T(Res.getTargetOpts().Triple);
auto Arch = T.getArch();
if (Arch == llvm::Triple::spir || Arch == llvm::Triple::spir64) {
Res.getDiagnosticOpts().Warnings.push_back("spir-compat");
}
return Success;
}
std::string CompilerInvocation::getModuleHash() const {
// Note: For QoI reasons, the things we use as a hash here should all be
// dumped via the -module-info flag.
using llvm::hash_code;
using llvm::hash_value;
using llvm::hash_combine;
// Start the signature with the compiler version.
// FIXME: We'd rather use something more cryptographically sound than
// CityHash, but this will do for now.
hash_code code = hash_value(getClangFullRepositoryVersion());
// Extend the signature with the language options
#define LANGOPT(Name, Bits, Default, Description) \
code = hash_combine(code, LangOpts->Name);
#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
code = hash_combine(code, static_cast<unsigned>(LangOpts->get##Name()));
#define BENIGN_LANGOPT(Name, Bits, Default, Description)
#define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
#include "clang/Basic/LangOptions.def"
for (StringRef Feature : LangOpts->ModuleFeatures)
code = hash_combine(code, Feature);
// Extend the signature with the target options.
code = hash_combine(code, TargetOpts->Triple, TargetOpts->CPU,
TargetOpts->ABI);
for (unsigned i = 0, n = TargetOpts->FeaturesAsWritten.size(); i != n; ++i)
code = hash_combine(code, TargetOpts->FeaturesAsWritten[i]);
// Extend the signature with preprocessor options.
const PreprocessorOptions &ppOpts = getPreprocessorOpts();
const HeaderSearchOptions &hsOpts = getHeaderSearchOpts();
code = hash_combine(code, ppOpts.UsePredefines, ppOpts.DetailedRecord);
for (std::vector<std::pair<std::string, bool/*isUndef*/>>::const_iterator
I = getPreprocessorOpts().Macros.begin(),
IEnd = getPreprocessorOpts().Macros.end();
Index: lib/Driver/ToolChains/Clang.cpp
===================================================================
--- lib/Driver/ToolChains/Clang.cpp
+++ lib/Driver/ToolChains/Clang.cpp
@@ -2584,34 +2584,42 @@
else
Args.ClaimAllArgs(options::OPT_fmodule_file);
+ // -fmodule-file-map specifies the file containing the module names to
+ // precompiled module files mapping.
+ //
+ if (HaveModules)
+ Args.AddAllArgs(CmdArgs, options::OPT_fmodule_file_map);
+ else
+ Args.ClaimAllArgs(options::OPT_fmodule_file_map);
+
// When building modules and generating crashdumps, we need to dump a module
// dependency VFS alongside the output.
if (HaveClangModules && C.isForDiagnostics()) {
SmallString<128> VFSDir(Output.getFilename());
llvm::sys::path::replace_extension(VFSDir, ".cache");
// Add the cache directory as a temp so the crash diagnostics pick it up.
C.addTempFile(Args.MakeArgString(VFSDir));
llvm::sys::path::append(VFSDir, "vfs");
CmdArgs.push_back("-module-dependency-dir");
CmdArgs.push_back(Args.MakeArgString(VFSDir));
}
if (HaveClangModules)
Args.AddLastArg(CmdArgs, options::OPT_fmodules_user_build_path);
// Pass through all -fmodules-ignore-macro arguments.
Args.AddAllArgs(CmdArgs, options::OPT_fmodules_ignore_macro);
Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_interval);
Args.AddLastArg(CmdArgs, options::OPT_fmodules_prune_after);
Args.AddLastArg(CmdArgs, options::OPT_fbuild_session_timestamp);
if (Arg *A = Args.getLastArg(options::OPT_fbuild_session_file)) {
if (Args.hasArg(options::OPT_fbuild_session_timestamp))
D.Diag(diag::err_drv_argument_not_allowed_with)
<< A->getAsString(Args) << "-fbuild-session-timestamp";
llvm::sys::fs::file_status Status;
if (llvm::sys::fs::status(A->getValue(), Status))
D.Diag(diag::err_drv_no_such_file) << A->getValue();
Index: include/clang/Driver/Options.td
===================================================================
--- include/clang/Driver/Options.td
+++ include/clang/Driver/Options.td
@@ -1127,6 +1127,9 @@
def fmodule_file : Joined<["-"], "fmodule-file=">,
Group<i_Group>, Flags<[DriverOption,CC1Option]>, MetaVarName<"[<name>=]<file>">,
HelpText<"Specify the mapping of module name to precompiled module file, or load a module file if name is omitted.">;
+def fmodule_file_map : Joined<["-"], "fmodule-file-map=">,
+ Group<i_Group>, Flags<[DriverOption,CC1Option]>, MetaVarName<"[<prefix>=]<file>">,
+ HelpText<"Read the mapping of module names to precompiled module files from <file>.">;
def fmodules_ignore_macro : Joined<["-"], "fmodules-ignore-macro=">, Group<f_Group>, Flags<[CC1Option]>,
HelpText<"Ignore the definition of the given macro when building and loading modules">;
def fmodules_decluse : Flag <["-"], "fmodules-decluse">, Group<f_Group>,
Index: include/clang/Basic/DiagnosticDriverKinds.td
===================================================================
--- include/clang/Basic/DiagnosticDriverKinds.td
+++ include/clang/Basic/DiagnosticDriverKinds.td
@@ -326,4 +326,6 @@
"unable to find a Visual Studio installation; "
"try running Clang from a developer command prompt">,
InGroup<DiagGroup<"msvc-not-found">>;
+
+def err_drv_invalid_module_file_map : Error<"invalid module file mapping '%0'">;
}
Index: docs/Modules.rst
===================================================================
--- docs/Modules.rst
+++ docs/Modules.rst
@@ -222,165 +222,175 @@
specified file also overrides this module's paths that might be embedded
in other precompiled module files.
+``-fmodule-file-map=[<prefix>=]<file>``
+ Read the mapping of module names to precompiled module files from file. If
+ the argument includes optional prefix, then only lines starting with this
+ string are considered (with the prefix itself ignored). Each mapping entry
+ should be in the same form as the ``-fmodule-file`` value with module
+ name. Leading and trailing whitespaces in the value as well as blank lines
+ are ignored. The line prefix can be used to store the mapping in an already
+ existing file, for example, as comments in makefile fragments produced by
+ the ``-M`` option family.
+
``-fprebuilt-module-path=<directory>``
Specify the path to the prebuilt modules. If specified, we will look for modules in this directory for a given top-level module name. We don't need a module map for loading prebuilt modules in this directory and the compiler will not try to rebuild these modules. This can be specified multiple times.
Module Semantics
================
Modules are modeled as if each submodule were a separate translation unit, and a module import makes names from the other translation unit visible. Each submodule starts with a new preprocessor state and an empty translation unit.
.. note::
This behavior is currently only approximated when building a module with submodules. Entities within a submodule that has already been built are visible when building later submodules in that module. This can lead to fragile modules that depend on the build order used for the submodules of the module, and should not be relied upon. This behavior is subject to change.
As an example, in C, this implies that if two structs are defined in different submodules with the same name, those two types are distinct types (but may be *compatible* types if their definitions match). In C++, two structs defined with the same name in different submodules are the *same* type, and must be equivalent under C++'s One Definition Rule.
.. note::
Clang currently only performs minimal checking for violations of the One Definition Rule.
If any submodule of a module is imported into any part of a program, the entire top-level module is considered to be part of the program. As a consequence of this, Clang may diagnose conflicts between an entity declared in an unimported submodule and an entity declared in the current translation unit, and Clang may inline or devirtualize based on knowledge from unimported submodules.
Macros
------
The C and C++ preprocessor assumes that the input text is a single linear buffer, but with modules this is not the case. It is possible to import two modules that have conflicting definitions for a macro (or where one ``#define``\s a macro and the other ``#undef``\ines it). The rules for handling macro definitions in the presence of modules are as follows:
* Each definition and undefinition of a macro is considered to be a distinct entity.
* Such entities are *visible* if they are from the current submodule or translation unit, or if they were exported from a submodule that has been imported.
* A ``#define X`` or ``#undef X`` directive *overrides* all definitions of ``X`` that are visible at the point of the directive.
* A ``#define`` or ``#undef`` directive is *active* if it is visible and no visible directive overrides it.
* A set of macro directives is *consistent* if it consists of only ``#undef`` directives, or if all ``#define`` directives in the set define the macro name to the same sequence of tokens (following the usual rules for macro redefinitions).
* If a macro name is used and the set of active directives is not consistent, the program is ill-formed. Otherwise, the (unique) meaning of the macro name is used.
For example, suppose:
* ``<stdio.h>`` defines a macro ``getc`` (and exports its ``#define``)
* ``<cstdio>`` imports the ``<stdio.h>`` module and undefines the macro (and exports its ``#undef``)
The ``#undef`` overrides the ``#define``, and a source file that imports both modules *in any order* will not see ``getc`` defined as a macro.
Module Map Language
===================
.. warning::
The module map language is not currently guaranteed to be stable between major revisions of Clang.
The module map language describes the mapping from header files to the
logical structure of modules. To enable support for using a library as
a module, one must write a ``module.modulemap`` file for that library. The
``module.modulemap`` file is placed alongside the header files themselves,
and is written in the module map language described below.
.. note::
For compatibility with previous releases, if a module map file named
``module.modulemap`` is not found, Clang will also search for a file named
``module.map``. This behavior is deprecated and we plan to eventually
remove it.
As an example, the module map file for the C standard library might look a bit like this:
.. parsed-literal::
module std [system] [extern_c] {
module assert {
textual header "assert.h"
header "bits/assert-decls.h"
export *
}
module complex {
header "complex.h"
export *
}
module ctype {
header "ctype.h"
export *
}
module errno {
header "errno.h"
header "sys/errno.h"
export *
}
module fenv {
header "fenv.h"
export *
}
// ...more headers follow...
}
Here, the top-level module ``std`` encompasses the whole C standard library. It has a number of submodules containing different parts of the standard library: ``complex`` for complex numbers, ``ctype`` for character types, etc. Each submodule lists one of more headers that provide the contents for that submodule. Finally, the ``export *`` command specifies that anything included by that submodule will be automatically re-exported.
Lexical structure
-----------------
Module map files use a simplified form of the C99 lexer, with the same rules for identifiers, tokens, string literals, ``/* */`` and ``//`` comments. The module map language has the following reserved words; all other C identifiers are valid identifiers.
.. parsed-literal::
``config_macros`` ``export`` ``private``
``conflict`` ``framework`` ``requires``
``exclude`` ``header`` ``textual``
``explicit`` ``link`` ``umbrella``
``extern`` ``module`` ``use``
Module map file
---------------
A module map file consists of a series of module declarations:
.. parsed-literal::
*module-map-file*:
*module-declaration**
Within a module map file, modules are referred to by a *module-id*, which uses periods to separate each part of a module's name:
.. parsed-literal::
*module-id*:
*identifier* ('.' *identifier*)*
Module declaration
------------------
A module declaration describes a module, including the headers that contribute to that module, its submodules, and other aspects of the module.
.. parsed-literal::
*module-declaration*:
``explicit``:sub:`opt` ``framework``:sub:`opt` ``module`` *module-id* *attributes*:sub:`opt` '{' *module-member** '}'
``extern`` ``module`` *module-id* *string-literal*
The *module-id* should consist of only a single *identifier*, which provides the name of the module being defined. Each module shall have a single definition.
The ``explicit`` qualifier can only be applied to a submodule, i.e., a module that is nested within another module. The contents of explicit submodules are only made available when the submodule itself was explicitly named in an import declaration or was re-exported from an imported module.
The ``framework`` qualifier specifies that this module corresponds to a Darwin-style framework. A Darwin-style framework (used primarily on Mac OS X and iOS) is contained entirely in directory ``Name.framework``, where ``Name`` is the name of the framework (and, therefore, the name of the module). That directory has the following layout:
.. parsed-literal::
Name.framework/
Modules/module.modulemap Module map for the framework
Headers/ Subdirectory containing framework headers
PrivateHeaders/ Subdirectory containing framework private headers
Frameworks/ Subdirectory containing embedded frameworks
Resources/ Subdirectory containing additional resources
Name Symbolic link to the shared library for the framework
The ``system`` attribute specifies that the module is a system module. When a system module is rebuilt, all of the module's headers will be considered system headers, which suppresses warnings. This is equivalent to placing ``#pragma GCC system_header`` in each of the module's headers. The form of attributes is described in the section Attributes_, below.
The ``extern_c`` attribute specifies that the module contains C code that can be used from within C++. When such a module is built for use in C++ code, all of the module's headers will be treated as if they were contained within an implicit ``extern "C"`` block. An import for a module with this attribute can appear within an ``extern "C"`` block. No other restrictions are lifted, however: the module currently cannot be imported within an ``extern "C"`` block in a namespace.
The ``no_undeclared_includes`` attribute specifies that the module can only reach non-modular headers and headers from used modules. Since some headers could be present in more than one search path and map to different modules in each path, this mechanism helps clang to find the right header, i.e., prefer the one for the current module or in a submodule instead of the first usual match in the search paths.
Modules can have a number of different kinds of members, each of which is described below:
.. parsed-literal::
*module-member*:
*requires-declaration*
*header-declaration*
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
