diff --git include/clang/Basic/DiagnosticLexKinds.td include/clang/Basic/DiagnosticLexKinds.td index 3c036a3..bfdef62 100644 --- include/clang/Basic/DiagnosticLexKinds.td +++ include/clang/Basic/DiagnosticLexKinds.td @@ -93,7 +93,13 @@ def ext_multichar_character_literal : ExtWarn< "multi-character character constant">, InGroup; def ext_four_char_character_literal : Extension< "multi-character character constant">, InGroup; - + + +def err_ucn_invalid_in_id : Error< + "character '%0' cannot be used as a universal character name " + "in an identifer">; +def err_ucn_invalid_at_start_id : Error< + "character '%0' cannot be used at the start of an identifer">; // Literal def ext_nonstandard_escape : Extension< diff --git include/clang/Lex/Lexer.h include/clang/Lex/Lexer.h index d36189f..a712f93 100644 --- include/clang/Lex/Lexer.h +++ include/clang/Lex/Lexer.h @@ -579,6 +579,8 @@ private: void cutOffLexing() { BufferPtr = BufferEnd; } bool isHexaLiteral(const char *Start, const LangOptions &LangOpts); + + static bool isUCNAfterSlash(const char *CurPtr, const LangOptions &LangOpts); }; diff --git include/clang/Lex/Token.h include/clang/Lex/Token.h index 06ff56e..e77264e 100644 --- include/clang/Lex/Token.h +++ include/clang/Lex/Token.h @@ -74,9 +74,10 @@ public: StartOfLine = 0x01, // At start of line or only after whitespace. LeadingSpace = 0x02, // Whitespace exists before this token. DisableExpand = 0x04, // This identifier may never be macro expanded. - NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. + NeedsCleaning = 0x08, // Contained an escaped newline or trigraph. LeadingEmptyMacro = 0x10, // Empty macro exists before this token. - HasUDSuffix = 0x20 // This string or character literal has a ud-suffix. + HasUDSuffix = 0x20, // This string or character literal has a ud-suffix. + HasUCN = 0x40 // This identifier contains a UCN. }; tok::TokenKind getKind() const { return (tok::TokenKind)Kind; } diff --git lib/Lex/Lexer.cpp lib/Lex/Lexer.cpp index 15b1061..c5e1f54 100644 --- lib/Lex/Lexer.cpp +++ lib/Lex/Lexer.cpp @@ -371,10 +371,12 @@ unsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, // NOTE: this has to be checked *before* testing for an IdentifierInfo. if (Tok.is(tok::raw_identifier)) TokStart = Tok.getRawIdentifierData(); - else if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { - // Just return the string from the identifier table, which is very quick. - Buffer = II->getNameStart(); - return II->getLength(); + else if (!(Tok.getFlags() & Token::HasUCN)) { + if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { + // Just return the string from the identifier table, which is very quick. + Buffer = II->getNameStart(); + return II->getLength(); + } } // NOTE: this can be checked even after testing for an IdentifierInfo. @@ -1376,7 +1378,6 @@ SourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, /// 2. If this is an escaped newline (potentially with whitespace between /// the backslash and newline), implicitly skip the newline and return /// the char after it. -/// 3. If this is a UCN, return it. FIXME: C++ UCN's? /// /// This handles the slow/uncommon case of the getCharAndSize method. Here we /// know that we can accumulate into Size, and that we have already incremented @@ -1501,6 +1502,31 @@ Slash: // Helper methods for lexing. //===----------------------------------------------------------------------===// +bool Lexer::isUCNAfterSlash(const char *CurPtr, const LangOptions &LangOpts) { + if (!LangOpts.CPlusPlus && !LangOpts.C99) + return false; + + unsigned CharSize; + char FirstChar = getCharAndSizeNoWarn(CurPtr, CharSize, LangOpts); + CurPtr += CharSize; + + unsigned NumHexDigits; + if (FirstChar == 'u') + NumHexDigits = 4; + else if (FirstChar == 'U') + NumHexDigits = 8; + else + return false; + + for (unsigned i = 0; i < NumHexDigits; ++i) { + if (!isxdigit(getCharAndSizeNoWarn(CurPtr, CharSize, LangOpts))) + return false; + CurPtr += CharSize; + } + + return true; +} + /// \brief Routine that indiscriminately skips bytes in the source file. void Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { BufferPtr += Bytes; @@ -1520,7 +1546,6 @@ void Lexer::LexIdentifier(Token &Result, const char *CurPtr) { // Fast path, no $,\,? in identifier found. '\' might be an escaped newline // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. - // FIXME: UCNs. // // TODO: Could merge these checks into a CharInfo flag to make the comparison // cheaper @@ -1561,16 +1586,18 @@ FinishIdentifier: CurPtr = ConsumeChar(CurPtr, Size, Result); C = getCharAndSize(CurPtr, Size); continue; - } else if (!isIdentifierBody(C)) { // FIXME: UCNs. - // Found end of identifier. - goto FinishIdentifier; + } else if (!isIdentifierBody(C)) { + if (C == '\\' && isUCNAfterSlash(CurPtr+Size, LangOpts)) + Result.setFlag(Token::HasUCN); + else + goto FinishIdentifier; } // Otherwise, this character is good, consume it. CurPtr = ConsumeChar(CurPtr, Size, Result); C = getCharAndSize(CurPtr, Size); - while (isIdentifierBody(C)) { // FIXME: UCNs. + while (isIdentifierBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); C = getCharAndSize(CurPtr, Size); } @@ -1595,7 +1622,7 @@ void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { unsigned Size; char C = getCharAndSize(CurPtr, Size); char PrevCh = 0; - while (isNumberBody(C)) { // FIXME: UCNs in ud-suffix. + while (isNumberBody(C)) { CurPtr = ConsumeChar(CurPtr, Size, Result); PrevCh = C; C = getCharAndSize(CurPtr, Size); @@ -2883,6 +2910,18 @@ LexNextToken: Kind = tok::unknown; break; + // Identifiers beginning with UCNs (C99 6.4.2.1, C++11 [lex.name]) + case '\\': + if (isUCNAfterSlash(CurPtr, LangOpts)) { + // Notify MIOpt that we read a non-whitespace/non-comment token. + MIOpt.ReadToken(); + Result.setFlag(Token::HasUCN); + return LexIdentifier(Result, CurPtr); + } + + Kind = tok::unknown; + break; + // C99 6.4.4: Character Constants. case '\'': // Notify MIOpt that we read a non-whitespace/non-comment token. @@ -3243,9 +3282,6 @@ LexNextToken: Kind = tok::unknown; break; - case '\\': - // FIXME: UCN's. - // FALL THROUGH. default: Kind = tok::unknown; break; diff --git lib/Lex/Preprocessor.cpp lib/Lex/Preprocessor.cpp index df2c98d..fc31ae0 100644 --- lib/Lex/Preprocessor.cpp +++ lib/Lex/Preprocessor.cpp @@ -27,6 +27,7 @@ #include "clang/Lex/Preprocessor.h" #include "MacroArgs.h" +#include "clang/Basic/ConvertUTF.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/TargetInfo.h" @@ -43,6 +44,7 @@ #include "clang/Lex/ScratchBuffer.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/SmallString.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/Support/Capacity.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" @@ -399,7 +401,7 @@ StringRef Preprocessor::getSpelling(const Token &Tok, SmallVectorImpl &Buffer, bool *Invalid) const { // NOTE: this has to be checked *before* testing for an IdentifierInfo. - if (Tok.isNot(tok::raw_identifier)) { + if (Tok.isNot(tok::raw_identifier) && !(Tok.getFlags() & Token::HasUCN)) { // Try the fast path. if (const IdentifierInfo *II = Tok.getIdentifierInfo()) return II->getName(); @@ -497,6 +499,88 @@ void Preprocessor::EndSourceFile() { // Lexer Event Handling. //===----------------------------------------------------------------------===// +static int hexDigitValue(char C) { + if (C >= '0' && C <= '9') return C-'0'; + if (C >= 'a' && C <= 'f') return C-'a'+10; + return C-'A'+10; +} + +namespace { + struct UCNCharRange { + unsigned Lower; + unsigned Upper; + }; + + // C11 D.1, C++11 [charname.allowed] + // FIXME: C99 and C++03 each have a different set of allowed UCNs. + UCNCharRange UCNAllowedCharRanges[] = { + // 1 + { 0x00A8, 0x00A8 }, { 0x00AA, 0x00AA }, { 0x00AD, 0x00AD }, + { 0x00AF, 0x00AF }, { 0x00B2, 0x00B5 }, { 0x00B7, 0x00BA }, + { 0x00BC, 0x00BE }, { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 }, + { 0x00F8, 0x00FF }, + // 2 + { 0x0100, 0x167F }, { 0x1681, 0x180D }, { 0x180F, 0x1FFF }, + // 3 + { 0x200B, 0x200D }, { 0x202A, 0x202E }, { 0x203F, 0x2040 }, + { 0x2054, 0x2054 }, { 0x2060, 0x206F }, + // 4 + { 0x2070, 0x218F }, { 0x2460, 0x24FF }, { 0x2776, 0x2793 }, + { 0x2C00, 0x2DFF }, { 0x2E80, 0x2FFF }, + // 5 + { 0x3004, 0x3007 }, { 0x3021, 0x302F }, { 0x3031, 0x303F }, + // 6 + { 0x3040, 0xD7FF }, + // 7 + { 0xF900, 0xFD3D }, { 0xFD40, 0xFDCF }, { 0xFDF0, 0xFE44 }, + { 0xFE47, 0xFFFD }, + // 8 + { 0x10000, 0x1FFFD }, { 0x20000, 0x2FFFD }, { 0x30000, 0x3FFFD }, + { 0x40000, 0x4FFFD }, { 0x50000, 0x5FFFD }, { 0x60000, 0x6FFFD }, + { 0x70000, 0x7FFFD }, { 0x80000, 0x8FFFD }, { 0x90000, 0x9FFFD }, + { 0xA0000, 0xAFFFD }, { 0xB0000, 0xBFFFD }, { 0xC0000, 0xCFFFD }, + { 0xD0000, 0xDFFFD }, { 0xE0000, 0xEFFFD } + }; +} + +static bool isAllowedIDChar(unsigned c) { + unsigned LowPoint = 0; + unsigned HighPoint = llvm::array_lengthof(UCNAllowedCharRanges); + + // Binary search the UCNAllowedCharRanges set. + while (HighPoint != LowPoint) { + unsigned MidPoint = (HighPoint + LowPoint) / 2; + if (c < UCNAllowedCharRanges[MidPoint].Lower) + HighPoint = MidPoint; + else if (c > UCNAllowedCharRanges[MidPoint].Upper) + LowPoint = MidPoint + 1; + else + return true; + } + + return false; +} + +static bool isAllowedInitiallyIDChar(unsigned c) { + // C11 D.2, C++11 [charname.disallowed] + // FIXME: C99 only forbids "digits", presumably as described in C99 Annex D. + // FIXME: C++03 does not forbid any initial characters. + return !(0x0300 <= c && c <= 0x036F) && + !(0x1DC0 <= c && c <= 0x1DFF) && + !(0x20D0 <= c && c <= 0x20FF) && + !(0xFE20 <= c && c <= 0xFE2F); +} + +static void appendCodePoint(unsigned Codepoint, + llvm::SmallVectorImpl &Str) { + char ResultBuf[4]; + char *ResultPtr = ResultBuf; + bool Res = ConvertCodePointToUTF8(Codepoint, ResultPtr); + (void)Res; + assert(Res && "Unexpected conversion failure"); + Str.append(ResultBuf, ResultPtr); +} + /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the /// identifier information for the token and install it into the token, /// updating the token kind accordingly. @@ -505,14 +589,63 @@ IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { // Look up this token, see if it is a macro, or if it is a language keyword. IdentifierInfo *II; - if (!Identifier.needsCleaning()) { + if (!Identifier.needsCleaning() && !(Identifier.getFlags() & Token::HasUCN)) { // No cleaning needed, just use the characters from the lexed buffer. II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(), - Identifier.getLength())); + Identifier.getLength())); } else { // Cleaning needed, alloca a buffer, clean into it, then use the buffer. SmallString<64> IdentifierBuffer; StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); + + SmallString<64> UCNIdentifierBuffer; + if (Identifier.getFlags() & Token::HasUCN) { + for (unsigned i = 0, e = CleanedStr.size(); i != e; ++i) { + if (CleanedStr[i] != '\\') { + UCNIdentifierBuffer.push_back(CleanedStr[i]); + continue; + } + + unsigned UCNVal; + unsigned NumChars; + if (CleanedStr[i+1] == 'u') { + UCNVal = (hexDigitValue(CleanedStr[i+2]) << 12) + + (hexDigitValue(CleanedStr[i+3]) << 8) + + (hexDigitValue(CleanedStr[i+4]) << 4) + + (hexDigitValue(CleanedStr[i+5])); + NumChars = 6; + } else { + assert(CleanedStr[i+1] == 'U'); + UCNVal = (hexDigitValue(CleanedStr[i+2]) << 28) + + (hexDigitValue(CleanedStr[i+3]) << 24) + + (hexDigitValue(CleanedStr[i+4]) << 20) + + (hexDigitValue(CleanedStr[i+5]) << 16) + + (hexDigitValue(CleanedStr[i+6]) << 12) + + (hexDigitValue(CleanedStr[i+7]) << 8) + + (hexDigitValue(CleanedStr[i+8]) << 4) + + (hexDigitValue(CleanedStr[i+9])); + NumChars = 10; + } + + if (!isAllowedIDChar(UCNVal)) { + StringRef CurCharacter = CleanedStr.substr(i, NumChars); + Diag(Identifier, diag::err_ucn_invalid_in_id) << CurCharacter; + UCNVal = 0xFFFD; + } + + if (UCNIdentifierBuffer.empty() && !isAllowedInitiallyIDChar(UCNVal)) { + StringRef CurCharacter = CleanedStr.substr(i, NumChars); + Diag(Identifier, diag::err_ucn_invalid_at_start_id) << CurCharacter; + UCNVal = 0xFFFD; + } + + appendCodePoint(UCNVal, UCNIdentifierBuffer); + i += NumChars - 1; + } + + CleanedStr = UCNIdentifierBuffer; + } + II = getIdentifierInfo(CleanedStr); } diff --git test/CXX/over/over.oper/over.literal/p8.cpp test/CXX/over/over.oper/over.literal/p8.cpp index 6f63610..70a1843 100644 --- test/CXX/over/over.oper/over.literal/p8.cpp +++ test/CXX/over/over.oper/over.literal/p8.cpp @@ -7,8 +7,7 @@ namespace std { void operator "" _km(long double); // ok string operator "" _i18n(const char*, std::size_t); // ok -// FIXME: This should be accepted once we support UCNs -template int operator "" \u03C0(); // ok, UCN for lowercase pi // expected-error {{expected identifier}} +template int operator "" \u03C0(); // ok, UCN for lowercase pi // expected-warning {{reserved}} float operator ""E(const char *); // expected-error {{invalid suffix on literal}} expected-warning {{reserved}} float operator " " B(const char *); // expected-error {{must be '""'}} expected-warning {{reserved}} string operator "" 5X(const char *, std::size_t); // expected-error {{expected identifier}} diff --git test/Preprocessor/ucn-pp-identifier.c test/Preprocessor/ucn-pp-identifier.c new file mode 100644 index 0000000..8fd96b1 --- /dev/null +++ test/Preprocessor/ucn-pp-identifier.c @@ -0,0 +1,72 @@ +// RUN: %clang_cc1 %s -fsyntax-only -std=c99 -pedantic -verify -Wundef +// RUN: %clang_cc1 %s -fsyntax-only -x c++ -pedantic -verify -Wundef + +#define \u00FC +#define a\u00FD() 0 +#ifndef \u00FC +#error "This should never happen" +#endif + +#if a\u00FD() +#error "This should never happen" +#endif + +#if a\U000000FD() +#error "This should never happen" +#endif + +// Check that we don't accept all uses of \u and \U as UCNs. +// (Again, sort of weird, but part of the rules) +#if \uarecool // expected-error {{invalid token at start of a preprocessor expression}} +#endif +#if \U0001000 // expected-error {{invalid token at start of a preprocessor expression}} +#endif + +// Make sure we reject disallowed UCNs +#define \ufffe // expected-error {{character '\ufffe' cannot be used as a universal character name in an identifer}} +#define \U10000000 // expected-error {{character '\U10000000' cannot be used as a universal character name in an identifer}} +#define \u0061 // expected-error {{character '\u0061' cannot be used as a universal character name in an identifer}} + +// FIXME: Not clear what our behavior should be here; \u0024 is "$". +#define a\u0024 // expected-error {{character '\u0024' cannot be used as a universal character name in an identifer}} + +#if \u0110 // expected-warning {{is not defined, evaluates to 0}} +#endif + + +#define \u0110 1 / 0 +#if \u0110 // expected-error {{division by zero in preprocessor expression}} +#endif + +#define STRINGIZE(X) # X + +extern int check_size[sizeof(STRINGIZE(\u0112)) == 3 ? 1 : -1]; + + +#ifndef __cplusplus + +#define newline_1_\u00F\ +C 1 +#define newline_2_\u00\ +F\ +C 1 +#define newline_3_\u\ +00\ +FC 1 +#define newline_4_\\ +u00FC 1 +#define newline_5_\\ +u\ +\ +0\ +0\ +F\ +C 1 + +#if (newline_1_\u00FC && newline_2_\u00FC && newline_3_\u00FC && \ + newline_4_\u00FC && newline_5_\u00FC) +#else +#error "Line splicing failed to produce UCNs" +#endif + +#endif