https://github.com/hvdijk updated https://github.com/llvm/llvm-project/pull/231223
>From 75949a692f03cb6b2f0b1377cade0582332fce1f Mon Sep 17 00:00:00 2001 From: Harald van Dijk <[email protected]> Date: Mon, 12 Oct 2026 17:49:03 +0100 Subject: [PATCH 1/4] Make isPrintable match Unicode As commented in the source code, the reason isPrintable had a special exception to consider U+00AD printable regardless of its Unicode classification was for Clang, and because of how it renders in terminals. But this is a function in general support library, part of LLVM, not Clang, and is also used for things that are not sent to terminals, so this is not appropriate for this function. It also introduced a discrepancy where U+00AD was simultaneously considered a printable character by isPrintable, and a formatting character by isFormatting. Move the U+00AD check into Clang instead. --- clang/lib/Basic/Diagnostic.cpp | 5 ++++- llvm/lib/Support/Unicode.cpp | 9 ++------- llvm/unittests/Support/YAMLIOTest.cpp | 20 ++++++++++---------- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp index 251c8ba6e13dbb..020a3f5738798d 100644 --- a/clang/lib/Basic/Diagnostic.cpp +++ b/clang/lib/Basic/Diagnostic.cpp @@ -1108,7 +1108,10 @@ static void EscapeStringForDiagnostic(StringRef Str, assert(Begin == CodepointEnd && "we must be further along in the string now"); - if (llvm::sys::unicode::isPrintable(CodepointValue) || + // Clang special cases 0x00AD (SOFT HYPHEN) which is rendered as an actual + // hyphen in most terminals. + if (CodepointValue == 0x00AD || + llvm::sys::unicode::isPrintable(CodepointValue) || (!ForCodepoint && llvm::sys::unicode::isFormatting(CodepointValue))) { OutStream << (ForCodepoint ? "'" : "") << StringRef(reinterpret_cast<const char *>(CodepointBegin), diff --git a/llvm/lib/Support/Unicode.cpp b/llvm/lib/Support/Unicode.cpp index 6c89270d29f761..5294c917a8b37d 100644 --- a/llvm/lib/Support/Unicode.cpp +++ b/llvm/lib/Support/Unicode.cpp @@ -25,15 +25,10 @@ extern const UnicodeCharSet::CharRanges GeneratedCombiningCharacterRanges; extern const UnicodeCharSet::CharRanges GeneratedDoubleWidthCharacterRanges; /// Unicode code points of the categories L, M, N, P, S and Zs are considered -/// printable. -/// In addition, U+00AD SOFT HYPHEN is also considered printable, as -/// it's actually displayed on most terminals. \return true if the character is -/// considered printable. +/// printable. \return true if the character is considered printable. bool isPrintable(int UCS) { static const UnicodeCharSet Printables(GeneratedPrintableRanges); - // Clang special cases 0x00AD (SOFT HYPHEN) which is rendered as an actual - // hyphen in most terminals. - return UCS == 0x00AD || Printables.contains(UCS); + return Printables.contains(UCS); } /// Unicode code points of the Cf category are considered diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp index 78e19c5ff4f2cc..1ceeec81181cb1 100644 --- a/llvm/unittests/Support/YAMLIOTest.cpp +++ b/llvm/unittests/Support/YAMLIOTest.cpp @@ -3024,16 +3024,16 @@ TEST(YAMLIO, TestEscaped) { TestEscaped("parameter 'параметр' is unused", "\"parameter 'параметр' is unused\""); - // String with embedded non-printable multibyte UTF-8 sequence (U+200B - // zero-width space). The thing to test here is that we emit a - // unicode-scalar level escape like \uNNNN (at the YAML level), and don't - // just pass the UTF-8 byte sequence through as with quoted printables. - { - const unsigned char foobar[10] = {'f', 'o', 'o', - 0xE2, 0x80, 0x8B, // UTF-8 of U+200B - 'b', 'a', 'r', - 0x0}; - TestEscaped((char const *)foobar, "\"foo\\u200Bbar\""); + // String with embedded non-printable multibyte UTF-8 sequence (U+00AD soft + // hyphen and U+200B zero-width space). The thing to test here is that we + // emit a unicode-scalar level escape like \uNNNN (at the YAML level), and + // don't just pass the UTF-8 byte sequence through as with quoted printables. + { + const unsigned char foobar[15] = { + 'f', 'o', 'o', 0xC2, 0xAD, // UTF-8 of U+00AD + 'b', 'a', 'r', 0xE2, 0x80, 0x8B, // UTF-8 of U+200B + 'b', 'a', 'z', 0x0}; + TestEscaped((char const *)foobar, "\"foo\\u00ADbar\\u200Bbaz\""); } } >From 9df0c3e0057709feb7f0b23360e30233de443c13 Mon Sep 17 00:00:00 2001 From: Harald van Dijk <[email protected]> Date: Mon, 12 Oct 2026 20:21:39 +0100 Subject: [PATCH 2/4] Update tests --- llvm/unittests/Support/UnicodeTest.cpp | 7 +++---- llvm/unittests/Support/YAMLIOTest.cpp | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/unittests/Support/UnicodeTest.cpp b/llvm/unittests/Support/UnicodeTest.cpp index bc562b7baea098..d88555512921db 100644 --- a/llvm/unittests/Support/UnicodeTest.cpp +++ b/llvm/unittests/Support/UnicodeTest.cpp @@ -33,9 +33,9 @@ TEST(Unicode, columnWidthUTF8) { // 00AD SOFT HYPHEN is displayed on most terminals as a space or a dash. Some // text editors display it only when a line is broken at it, some use it as a - // line-break hint, but don't display. We choose terminal-oriented + // line-break hint, but don't display. We do not take any particular // interpretation. - EXPECT_EQ(1, columnWidthUTF8("\302\255")); + EXPECT_EQ(-1, columnWidthUTF8("\302\255")); EXPECT_EQ(0, columnWidthUTF8("\314\200")); // 0300 COMBINING GRAVE ACCENT EXPECT_EQ(1, columnWidthUTF8("\340\270\201")); // 0E01 THAI CHARACTER KO KAI @@ -81,8 +81,7 @@ TEST(Unicode, isPrintable) { EXPECT_FALSE(isPrintable(0x9F)); EXPECT_TRUE(isPrintable(0xAC)); - EXPECT_TRUE(isPrintable(0xAD)); // SOFT HYPHEN is displayed on most terminals - // as either a space or a dash. + EXPECT_FALSE(isPrintable(0xAD)); EXPECT_TRUE(isPrintable(0xAE)); EXPECT_TRUE(isPrintable(0x0377)); // GREEK SMALL LETTER PAMPHYLIAN DIGAMMA diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp index 1ceeec81181cb1..f7630f9e93fc8b 100644 --- a/llvm/unittests/Support/YAMLIOTest.cpp +++ b/llvm/unittests/Support/YAMLIOTest.cpp @@ -3033,7 +3033,7 @@ TEST(YAMLIO, TestEscaped) { 'f', 'o', 'o', 0xC2, 0xAD, // UTF-8 of U+00AD 'b', 'a', 'r', 0xE2, 0x80, 0x8B, // UTF-8 of U+200B 'b', 'a', 'z', 0x0}; - TestEscaped((char const *)foobar, "\"foo\\u00ADbar\\u200Bbaz\""); + TestEscaped((char const *)foobar, "\"foo\\xADbar\\u200Bbaz\""); } } >From 5072778e4d02363c8f47c6d9589827ad51fa4381 Mon Sep 17 00:00:00 2001 From: Harald van Dijk <[email protected]> Date: Wed, 14 Oct 2026 02:51:06 +0100 Subject: [PATCH 3/4] Change isPrintable instead, and update Clang to match --- clang/lib/Basic/Diagnostic.cpp | 8 +- clang/lib/Frontend/TextDiagnostic.cpp | 8 +- clang/test/Lexer/unicode.c | 10 +- clang/test/Preprocessor/ucn-allowed-chars.c | 6 +- clang/test/Preprocessor/ucn-pp-identifier.c | 2 +- .../warn-implicit-unicode-conversions.cpp | 2 +- clang/unittests/Basic/DiagnosticTest.cpp | 2 +- llvm/include/llvm/Support/Unicode.h | 13 +- llvm/lib/Support/Unicode.cpp | 18 +- llvm/lib/Support/UnicodeCharSetsGenerated.cpp | 503 +++++++++--------- llvm/unittests/Support/UnicodeTest.cpp | 11 +- llvm/unittests/Support/YAMLIOTest.cpp | 19 +- .../UnicodeData/UnicodeCharSetsGenerator.cpp | 18 +- 13 files changed, 326 insertions(+), 294 deletions(-) diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp index 020a3f5738798d..49918b8f90133f 100644 --- a/clang/lib/Basic/Diagnostic.cpp +++ b/clang/lib/Basic/Diagnostic.cpp @@ -1108,11 +1108,8 @@ static void EscapeStringForDiagnostic(StringRef Str, assert(Begin == CodepointEnd && "we must be further along in the string now"); - // Clang special cases 0x00AD (SOFT HYPHEN) which is rendered as an actual - // hyphen in most terminals. - if (CodepointValue == 0x00AD || - llvm::sys::unicode::isPrintable(CodepointValue) || - (!ForCodepoint && llvm::sys::unicode::isFormatting(CodepointValue))) { + // Do not let through control characters which could mess up the terminal. + if (!llvm::sys::unicode::isControl(CodepointValue)) { OutStream << (ForCodepoint ? "'" : "") << StringRef(reinterpret_cast<const char *>(CodepointBegin), std::distance(CodepointBegin, CodepointEnd)) @@ -1120,7 +1117,6 @@ static void EscapeStringForDiagnostic(StringRef Str, if (!ForCodepoint) continue; } - // Unprintable code point. OutStream << (ForCodepoint ? "" : "<") << "U+" << llvm::format_hex_no_prefix(CodepointValue, 4, true) << (ForCodepoint ? "" : ">"); diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp index d1f44493b544ef..07cb185752877d 100644 --- a/clang/lib/Frontend/TextDiagnostic.cpp +++ b/clang/lib/Frontend/TextDiagnostic.cpp @@ -18,6 +18,7 @@ #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Locale.h" #include "llvm/Support/Path.h" +#include "llvm/Support/Unicode.h" #include <algorithm> #include <optional> @@ -178,8 +179,11 @@ printableTextForNextCharacter(StringRef SourceLine, size_t *I, (*I) += (Begin - OriginalBegin); - // Valid, multi-byte, printable UTF8 character. - if (llvm::sys::locale::isPrint(C)) + // Do not let through characters we cannot determine the width of, as they + // break caret diagnostics. + StringRef Codepoint(reinterpret_cast<const char *>(OriginalBegin), + std::distance(OriginalBegin, Begin)); + if (llvm::sys::unicode::columnWidthUTF8(Codepoint) >= 0) return std::make_pair(SmallString<16>(OriginalBegin, End), true); // Valid but not printable. diff --git a/clang/test/Lexer/unicode.c b/clang/test/Lexer/unicode.c index 963eca04e0908c..140c1c23b0db8d 100644 --- a/clang/test/Lexer/unicode.c +++ b/clang/test/Lexer/unicode.c @@ -88,7 +88,7 @@ extern int 👷; // expected-error {{unexpected character '👷' U+1F477}} \ extern int 👷♀; // expected-warning {{declaration does not declare anything}} \ expected-error {{unexpected character '👷' U+1F477}} \ - expected-error {{character U+200D not allowed at the start of an identifier}} \ + expected-error {{character '' U+200D not allowed at the start of an identifier}} \ expected-error {{unexpected character '♀' U+2640}} #else @@ -106,10 +106,10 @@ int *n꞉꞉v = &n;; // expected-warning 2{{treating character '꞉' U+A789 as int v＝［＝］（auto）｛return～x；｝（）; // expected-warning 12{{treating character}} int xx; -// expected-warning@-1 {{identifier contains character U+2060 that is invisible in some environments}} -// expected-warning@-2 {{identifier contains character U+FEFF that is invisible in some environments}} -// expected-warning@-3 {{identifier contains character U+200D that is invisible in some environments}} -int foobar = 0; // expected-warning {{identifier contains character U+200B that is invisible in some environments}} +// expected-warning@-1 {{identifier contains character '' U+2060 that is invisible in some environments}} +// expected-warning@-2 {{identifier contains character '' U+FEFF that is invisible in some environments}} +// expected-warning@-3 {{identifier contains character '' U+200D that is invisible in some environments}} +int foobar = 0; // expected-warning {{identifier contains character '' U+200B that is invisible in some environments}} int x = foobar; // expected-error {{undeclared identifier}} int ∣foo; // expected-error {{unexpected character '∣' U+2223}} diff --git a/clang/test/Preprocessor/ucn-allowed-chars.c b/clang/test/Preprocessor/ucn-allowed-chars.c index 8a6c40a3ecff59..8c8cddad5799f1 100644 --- a/clang/test/Preprocessor/ucn-allowed-chars.c +++ b/clang/test/Preprocessor/ucn-allowed-chars.c @@ -39,7 +39,7 @@ extern char \u0D61; // C99, C11, C++03, C++11 #if __cplusplus // expected-error@10 {{character '΄' U+0384 not allowed in an identifier}} -// expected-error@12 {{character U+FFFF not allowed in an identifier}} +// expected-error@12 {{character '�' U+FFFF not allowed in an identifier}} // expected-error@18 {{expected unqualified-id}} # if __cplusplus >= 201103L // C++11 @@ -57,7 +57,7 @@ extern char \u0D61; // C99, C11, C++03, C++11 // C23 // expected-warning@8 {{using this character in an identifier is incompatible with C99}} // expected-error@10 {{character '΄' U+0384 not allowed in an identifier}} -// expected-error@12 {{character U+FFFF not allowed in an identifier}} +// expected-error@12 {{character '�' U+FFFF not allowed in an identifier}} // expected-error@18 {{expected identifier}} // expected-error@19 {{expected identifier}} // expected-error@33 {{invalid universal character}} @@ -65,7 +65,7 @@ extern char \u0D61; // C99, C11, C++03, C++11 // C11 // expected-warning@8 {{using this character in an identifier is incompatible with C99}} // expected-warning@10 {{using this character in an identifier is incompatible with C99}} -// expected-error@12 {{character U+FFFF not allowed in an identifier}} +// expected-error@12 {{character '�' U+FFFF not allowed in an identifier}} // expected-warning@18 {{starting an identifier with this character is incompatible with C99}} // expected-error@19 {{expected identifier}} // expected-error@33 {{invalid universal character}} diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c index 0311d3eb29d1bd..2eeed16f3ced6f 100644 --- a/clang/test/Preprocessor/ucn-pp-identifier.c +++ b/clang/test/Preprocessor/ucn-pp-identifier.c @@ -161,4 +161,4 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>; #endif // GH64161 -int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character U+202D not allowed in an identifier}} +int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character '' U+202D not allowed in an identifier}} diff --git a/clang/test/SemaCXX/warn-implicit-unicode-conversions.cpp b/clang/test/SemaCXX/warn-implicit-unicode-conversions.cpp index 80420d836a55d6..202e18ab97a1c3 100644 --- a/clang/test/SemaCXX/warn-implicit-unicode-conversions.cpp +++ b/clang/test/SemaCXX/warn-implicit-unicode-conversions.cpp @@ -24,7 +24,7 @@ void test(char8_t u8, char16_t u16, char32_t u32) { c8(char16_t(0x7f)); c8(char16_t(0x80)); // expected-warning {{implicit conversion from 'char16_t' to 'char8_t' changes the meaning of the code point U+0080}} c8(char16_t(0xD800)); // expected-warning {{implicit conversion from 'char16_t' to 'char8_t' changes the meaning of the code unit <0xD800>}} - c8(char16_t(0xE000)); // expected-warning {{implicit conversion from 'char16_t' to 'char8_t' changes the meaning of the code point U+E000}} + c8(char16_t(0xE000)); // expected-warning {{implicit conversion from 'char16_t' to 'char8_t' changes the meaning of the code point '' U+E000}} c16(char32_t(0x7f)); diff --git a/clang/unittests/Basic/DiagnosticTest.cpp b/clang/unittests/Basic/DiagnosticTest.cpp index 793431bbbe1548..8ae1cbd2646a00 100644 --- a/clang/unittests/Basic/DiagnosticTest.cpp +++ b/clang/unittests/Basic/DiagnosticTest.cpp @@ -460,7 +460,7 @@ TEST(EscapeSingleCodepointForDiagnosticTest, nonPrintableDisplaysNoQuoted) { EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\n'), "U+000A"); EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\0'), "U+0000"); EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\x1B'), "U+001B"); - EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\uFEFF'), "U+FEFF"); + EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\uFEFF'), "'' U+FEFF"); } TEST(EscapeSingleCodepointForDiagnosticTest, nonScalarValues) { diff --git a/llvm/include/llvm/Support/Unicode.h b/llvm/include/llvm/Support/Unicode.h index 7c790da7b647d7..a82a71cce7edb7 100644 --- a/llvm/include/llvm/Support/Unicode.h +++ b/llvm/include/llvm/Support/Unicode.h @@ -30,15 +30,16 @@ enum ColumnWidthErrors { ErrorNonPrintableCharacter = -1 }; -/// Determines if a character is likely to be displayed correctly on the -/// terminal. Exact implementation would have to depend on the specific -/// terminal, so we define the semantic that should be suitable for generic case -/// of a terminal capable to output Unicode characters. -/// -/// Printable codepoints are those in the categories L, M, N, P, S and Zs +/// Determine whether a character can be printed. The definition of printable +/// follows ISO/IEC 30112, updated to apply to more recent Unicode versions. +/// Nonprintable code points are space, those in the categories Cc, Cs, Zl, Zp, +/// and non-characters. Other code points are printable. /// \return true if the character is considered printable. LLVM_ABI bool isPrintable(int UCS); +// Formatting codepoints are codepoints in the Cc category. +LLVM_ABI bool isControl(int UCS); + // Formatting codepoints are codepoints in the Cf category. LLVM_ABI bool isFormatting(int UCS); diff --git a/llvm/lib/Support/Unicode.cpp b/llvm/lib/Support/Unicode.cpp index 5294c917a8b37d..d839ae55548efe 100644 --- a/llvm/lib/Support/Unicode.cpp +++ b/llvm/lib/Support/Unicode.cpp @@ -20,6 +20,7 @@ namespace sys { namespace unicode { extern const UnicodeCharSet::CharRanges GeneratedPrintableRanges; +extern const UnicodeCharSet::CharRanges GeneratedControlCharacterRanges; extern const UnicodeCharSet::CharRanges GeneratedFormatCharacterRanges; extern const UnicodeCharSet::CharRanges GeneratedCombiningCharacterRanges; extern const UnicodeCharSet::CharRanges GeneratedDoubleWidthCharacterRanges; @@ -31,6 +32,13 @@ bool isPrintable(int UCS) { return Printables.contains(UCS); } +/// Unicode code points of the Cc category are considered +/// control characters. +bool isControl(int UCS) { + static const UnicodeCharSet Control(GeneratedControlCharacterRanges); + return Control.contains(UCS); +} + /// Unicode code points of the Cf category are considered /// formatting characters. bool isFormatting(int UCS) { @@ -46,13 +54,21 @@ bool isFormatting(int UCS) { /// \return Character width: /// * ErrorNonPrintableCharacter (-1) for non-printable characters (as /// identified by isPrintable); -/// * 0 for non-spacing and enclosing combining marks; +/// * 0 for non-spacing and enclosing combining marks, and for formatting +/// characters (as identified by isFormatting), with the exception of +/// U+00AD; /// * 2 for CJK characters excluding halfwidth forms; /// * 1 for all remaining characters. static inline int charWidth(int UCS) { if (!isPrintable(UCS)) return ErrorNonPrintableCharacter; + if (isFormatting(UCS)) { + // Special case 0x00AD (SOFT HYPHEN) which is rendered as an actual hyphen + // in most terminals. + return UCS == 0x00AD ? 1 : 0; + } + static const UnicodeCharSet CombiningCharacters( GeneratedCombiningCharacterRanges); diff --git a/llvm/lib/Support/UnicodeCharSetsGenerated.cpp b/llvm/lib/Support/UnicodeCharSetsGenerated.cpp index 0a30fe11c2ca9a..351a8de06ed218 100644 --- a/llvm/lib/Support/UnicodeCharSetsGenerated.cpp +++ b/llvm/lib/Support/UnicodeCharSetsGenerated.cpp @@ -18,258 +18,265 @@ using CharRanges = llvm::sys::UnicodeCharSet::CharRanges; namespace llvm::sys::unicode { -// General_Category=L|M|N|P|S|Zs +// General_Category!=Cc|Cs|Zl|Zp static constexpr UnicodeCharRange GeneratedPrintableRangesData[] = { - {0x0020, 0x007E}, {0x00A0, 0x00AC}, {0x00AE, 0x0377}, - {0x037A, 0x037F}, {0x0384, 0x038A}, {0x038C, 0x038C}, - {0x038E, 0x03A1}, {0x03A3, 0x052F}, {0x0531, 0x0556}, - {0x0558, 0x058F}, {0x0591, 0x05C9}, {0x05D0, 0x05EA}, - {0x05EF, 0x05F4}, {0x0606, 0x061B}, {0x061D, 0x06DC}, - {0x06DE, 0x070D}, {0x0710, 0x074A}, {0x074D, 0x07B1}, - {0x07C0, 0x07FA}, {0x07FD, 0x082D}, {0x0830, 0x083E}, - {0x0840, 0x085B}, {0x085E, 0x085E}, {0x0860, 0x086A}, - {0x0870, 0x088F}, {0x0897, 0x08E1}, {0x08E3, 0x0983}, - {0x0985, 0x098C}, {0x098F, 0x0990}, {0x0993, 0x09A8}, - {0x09AA, 0x09B0}, {0x09B2, 0x09B2}, {0x09B6, 0x09B9}, - {0x09BC, 0x09C4}, {0x09C7, 0x09C8}, {0x09CB, 0x09CE}, - {0x09D7, 0x09D7}, {0x09DC, 0x09DD}, {0x09DF, 0x09E3}, - {0x09E6, 0x09FE}, {0x0A01, 0x0A03}, {0x0A05, 0x0A0A}, - {0x0A0F, 0x0A10}, {0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, - {0x0A32, 0x0A33}, {0x0A35, 0x0A36}, {0x0A38, 0x0A39}, - {0x0A3C, 0x0A3C}, {0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, - {0x0A4B, 0x0A4D}, {0x0A51, 0x0A51}, {0x0A59, 0x0A5C}, - {0x0A5E, 0x0A5E}, {0x0A66, 0x0A76}, {0x0A81, 0x0A83}, - {0x0A85, 0x0A8D}, {0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, - {0x0AAA, 0x0AB0}, {0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, - {0x0ABC, 0x0AC5}, {0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD}, - {0x0AD0, 0x0AD0}, {0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1}, - {0x0AF9, 0x0AFF}, {0x0B01, 0x0B03}, {0x0B05, 0x0B0C}, - {0x0B0F, 0x0B10}, {0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, - {0x0B32, 0x0B33}, {0x0B35, 0x0B39}, {0x0B3C, 0x0B44}, - {0x0B47, 0x0B48}, {0x0B4B, 0x0B4D}, {0x0B53, 0x0B57}, - {0x0B5C, 0x0B5D}, {0x0B5F, 0x0B63}, {0x0B66, 0x0B77}, - {0x0B82, 0x0B83}, {0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, - {0x0B92, 0x0B95}, {0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, - {0x0B9E, 0x0B9F}, {0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, - {0x0BAE, 0x0BB9}, {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, - {0x0BCA, 0x0BCD}, {0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, - {0x0BE6, 0x0BFA}, {0x0C00, 0x0C0C}, {0x0C0E, 0x0C10}, - {0x0C12, 0x0C28}, {0x0C2A, 0x0C39}, {0x0C3C, 0x0C44}, - {0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, - {0x0C58, 0x0C5A}, {0x0C5C, 0x0C5D}, {0x0C60, 0x0C63}, - {0x0C66, 0x0C6F}, {0x0C77, 0x0C8C}, {0x0C8E, 0x0C90}, - {0x0C92, 0x0CA8}, {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, - {0x0CBC, 0x0CC4}, {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, - {0x0CD5, 0x0CD6}, {0x0CDC, 0x0CDE}, {0x0CE0, 0x0CE3}, - {0x0CE6, 0x0CEF}, {0x0CF1, 0x0CF3}, {0x0D00, 0x0D0C}, - {0x0D0E, 0x0D10}, {0x0D12, 0x0D44}, {0x0D46, 0x0D48}, - {0x0D4A, 0x0D4F}, {0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, - {0x0D81, 0x0D83}, {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, - {0x0DB3, 0x0DBB}, {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, - {0x0DCA, 0x0DCA}, {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, - {0x0DD8, 0x0DDF}, {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, - {0x0E01, 0x0E3A}, {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, - {0x0E84, 0x0E84}, {0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3}, - {0x0EA5, 0x0EA5}, {0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4}, - {0x0EC6, 0x0EC6}, {0x0EC8, 0x0ECE}, {0x0ED0, 0x0ED9}, - {0x0EDC, 0x0EDF}, {0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, - {0x0F71, 0x0F97}, {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, - {0x0FCE, 0x0FDA}, {0x1000, 0x10C5}, {0x10C7, 0x10C7}, - {0x10CD, 0x10CD}, {0x10D0, 0x1248}, {0x124A, 0x124D}, - {0x1250, 0x1256}, {0x1258, 0x1258}, {0x125A, 0x125D}, - {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, - {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, - {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, {0x12D8, 0x1310}, - {0x1312, 0x1315}, {0x1318, 0x135A}, {0x135D, 0x137C}, - {0x1380, 0x1399}, {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, - {0x1400, 0x169C}, {0x16A0, 0x16F8}, {0x1700, 0x1715}, - {0x171F, 0x1736}, {0x1740, 0x1753}, {0x1760, 0x176C}, - {0x176E, 0x1770}, {0x1772, 0x1773}, {0x1780, 0x17DD}, - {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1800, 0x180D}, - {0x180F, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x18AA}, - {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1920, 0x192B}, - {0x1930, 0x193B}, {0x1940, 0x1940}, {0x1944, 0x196D}, - {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, - {0x19D0, 0x19DA}, {0x19DE, 0x1A1B}, {0x1A1E, 0x1A5E}, - {0x1A60, 0x1A7C}, {0x1A7F, 0x1A89}, {0x1A90, 0x1A99}, - {0x1AA0, 0x1AAD}, {0x1AB0, 0x1AF0}, {0x1B00, 0x1B4C}, - {0x1B4E, 0x1BF3}, {0x1BFC, 0x1C37}, {0x1C3B, 0x1C49}, - {0x1C4D, 0x1C8A}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CC7}, - {0x1CD0, 0x1CFA}, {0x1D00, 0x1F15}, {0x1F18, 0x1F1D}, - {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, - {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, - {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, {0x1FB6, 0x1FC4}, - {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB}, {0x1FDD, 0x1FEF}, - {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE}, {0x2000, 0x200A}, - {0x2010, 0x2027}, {0x202F, 0x205F}, {0x2070, 0x2071}, - {0x2074, 0x20C4}, {0x20D0, 0x20F0}, {0x2100, 0x218B}, - {0x2190, 0x2429}, {0x2440, 0x244A}, {0x2460, 0x2B73}, - {0x2B76, 0x2CF3}, {0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, - {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, - {0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, - {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, - {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, - {0x2DE0, 0x2E5D}, {0x2E60, 0x2E63}, {0x2E80, 0x2E99}, - {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x303F}, - {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312F}, - {0x3131, 0x318E}, {0x3190, 0x31E5}, {0x31EF, 0x321E}, - {0x3220, 0xA48C}, {0xA490, 0xA4C6}, {0xA4D0, 0xA62B}, - {0xA640, 0xA6F7}, {0xA700, 0xA7DD}, {0xA7E2, 0xA7E2}, - {0xA7F1, 0xA82C}, {0xA830, 0xA839}, {0xA840, 0xA877}, - {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, {0xA8E0, 0xA953}, - {0xA95F, 0xA97C}, {0xA980, 0xA9CD}, {0xA9CF, 0xA9D9}, - {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36}, {0xAA40, 0xAA4D}, - {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2}, {0xAADB, 0xAAF6}, - {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, - {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB6D}, - {0xAB70, 0xABED}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, - {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, - {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, - {0xFB1D, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, - {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFDCF}, - {0xFDF0, 0xFE19}, {0xFE20, 0xFE52}, {0xFE54, 0xFE66}, - {0xFE68, 0xFE6B}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, - {0xFF01, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, - {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0xFFE0, 0xFFE6}, - {0xFFE8, 0xFFEE}, {0xFFFC, 0xFFFD}, {0x10000, 0x1000B}, - {0x1000D, 0x10026}, {0x10028, 0x1003A}, {0x1003C, 0x1003D}, - {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, - {0x10100, 0x10102}, {0x10107, 0x10133}, {0x10137, 0x1018E}, - {0x10190, 0x1019C}, {0x101A0, 0x101A0}, {0x101D0, 0x101FD}, - {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E0, 0x102FB}, - {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x1037A}, - {0x10380, 0x1039D}, {0x1039F, 0x103C3}, {0x103C8, 0x103D5}, - {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, - {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, - {0x1056F, 0x1057A}, {0x1057C, 0x1058A}, {0x1058C, 0x10592}, - {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, - {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, {0x105C0, 0x105F3}, - {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, - {0x10780, 0x10785}, {0x10787, 0x107B0}, {0x107B2, 0x107BF}, - {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, - {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, - {0x10857, 0x1089E}, {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, - {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x1091F, 0x10939}, - {0x1093F, 0x10959}, {0x10980, 0x109B7}, {0x109BC, 0x109CF}, - {0x109D2, 0x10A03}, {0x10A05, 0x10A06}, {0x10A0C, 0x10A13}, - {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, {0x10A38, 0x10A3A}, - {0x10A3F, 0x10A48}, {0x10A50, 0x10A58}, {0x10A60, 0x10A9F}, - {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6}, {0x10B00, 0x10B35}, - {0x10B39, 0x10B55}, {0x10B58, 0x10B72}, {0x10B78, 0x10B91}, - {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, - {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, {0x10CFA, 0x10D27}, - {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, {0x10D69, 0x10D85}, - {0x10D8E, 0x10D8F}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, - {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, {0x10EC2, 0x10EC7}, - {0x10EC9, 0x10EEE}, {0x10EF0, 0x10F27}, {0x10F30, 0x10F59}, - {0x10F70, 0x10F89}, {0x10FB0, 0x10FCB}, {0x10FE0, 0x10FF6}, - {0x11000, 0x1104D}, {0x11052, 0x11075}, {0x1107F, 0x110BC}, - {0x110BE, 0x110C2}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, - {0x11100, 0x11134}, {0x11136, 0x11147}, {0x11150, 0x11176}, - {0x11180, 0x111DF}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, - {0x11213, 0x11241}, {0x11280, 0x11286}, {0x11288, 0x11288}, - {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, {0x1129F, 0x112A9}, - {0x112B0, 0x112EA}, {0x112F0, 0x112F9}, {0x11300, 0x11303}, - {0x11305, 0x1130C}, {0x1130F, 0x11310}, {0x11313, 0x11328}, - {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, - {0x1133B, 0x11344}, {0x11347, 0x11348}, {0x1134B, 0x1134D}, - {0x11350, 0x11350}, {0x11357, 0x11357}, {0x1135D, 0x11363}, - {0x11366, 0x1136C}, {0x11370, 0x11374}, {0x11380, 0x11389}, - {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, {0x11390, 0x113B5}, - {0x113B7, 0x113C0}, {0x113C2, 0x113C2}, {0x113C5, 0x113C5}, - {0x113C7, 0x113CA}, {0x113CC, 0x113D5}, {0x113D7, 0x113D8}, - {0x113E1, 0x113E2}, {0x11400, 0x1145B}, {0x1145D, 0x11461}, - {0x11480, 0x114C7}, {0x114D0, 0x114D9}, {0x11580, 0x115B5}, - {0x115B8, 0x115DD}, {0x11600, 0x11644}, {0x11650, 0x11659}, - {0x11660, 0x1166C}, {0x11680, 0x116B9}, {0x116C0, 0x116C9}, - {0x116D0, 0x116E3}, {0x11700, 0x1171A}, {0x1171D, 0x1172B}, - {0x11730, 0x11746}, {0x11800, 0x1183B}, {0x118A0, 0x118F2}, - {0x118FF, 0x11906}, {0x11909, 0x11909}, {0x1190C, 0x11913}, - {0x11915, 0x11916}, {0x11918, 0x11935}, {0x11937, 0x11938}, - {0x1193B, 0x11946}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, - {0x119AA, 0x119D7}, {0x119DA, 0x119E4}, {0x11A00, 0x11A47}, - {0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8}, {0x11B00, 0x11B0A}, - {0x11B60, 0x11B67}, {0x11BC0, 0x11BE1}, {0x11BF0, 0x11BF9}, - {0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, {0x11C38, 0x11C45}, - {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, {0x11C92, 0x11CA7}, - {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, - {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, {0x11D3C, 0x11D3D}, - {0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, - {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, {0x11D90, 0x11D91}, - {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11DB0, 0x11DDB}, - {0x11DE0, 0x11DE9}, {0x11DF0, 0x11DF1}, {0x11EE0, 0x11EF8}, - {0x11F00, 0x11F10}, {0x11F12, 0x11F3A}, {0x11F3E, 0x11F5A}, - {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, {0x11FFF, 0x12399}, - {0x12400, 0x12543}, {0x12550, 0x12686}, {0x12F90, 0x12FF2}, - {0x13000, 0x1342F}, {0x13440, 0x13455}, {0x13460, 0x143FA}, - {0x14400, 0x14646}, {0x16100, 0x16139}, {0x16800, 0x16A38}, - {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16ABE}, - {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, - {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, - {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16D40, 0x16D79}, - {0x16E40, 0x16E9A}, {0x16EA0, 0x16EB8}, {0x16EBB, 0x16ED3}, - {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F}, - {0x16FE0, 0x16FE4}, {0x16FF0, 0x16FF6}, {0x17000, 0x18CDA}, - {0x18CFF, 0x18D20}, {0x18D80, 0x18DF2}, {0x18E00, 0x19191}, - {0x191A0, 0x191D2}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, - {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B128}, {0x1B132, 0x1B132}, - {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B168}, - {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, - {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BC9F}, - {0x1CC00, 0x1CCFC}, {0x1CD00, 0x1CEB3}, {0x1CEBA, 0x1CED0}, - {0x1CED2, 0x1CED4}, {0x1CEDD, 0x1CEFD}, {0x1CF00, 0x1CF2D}, - {0x1CF30, 0x1CF46}, {0x1CF50, 0x1CFC3}, {0x1D000, 0x1D0F5}, - {0x1D100, 0x1D172}, {0x1D17B, 0x1D245}, {0x1D250, 0x1D281}, - {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D300, 0x1D356}, - {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, - {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, - {0x1D4A9, 0x1D4AC}, {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, - {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, - {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, - {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, - {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A6}, {0x1D6A8, 0x1D7CB}, - {0x1D7CE, 0x1DA8B}, {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, - {0x1DB00, 0x1DB1C}, {0x1DF00, 0x1DF81}, {0x1DF90, 0x1DF96}, - {0x1DFCD, 0x1E006}, {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, - {0x1E023, 0x1E024}, {0x1E026, 0x1E02A}, {0x1E030, 0x1E06D}, - {0x1E08F, 0x1E08F}, {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, - {0x1E140, 0x1E149}, {0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE}, - {0x1E2C0, 0x1E2F9}, {0x1E2FF, 0x1E2FF}, {0x1E4D0, 0x1E4F9}, - {0x1E5D0, 0x1E5FA}, {0x1E5FF, 0x1E5FF}, {0x1E6C0, 0x1E6DE}, - {0x1E6E0, 0x1E6F5}, {0x1E6FE, 0x1E6FF}, {0x1E7E0, 0x1E7E6}, - {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, - {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8D6}, {0x1E900, 0x1E94B}, - {0x1E950, 0x1E959}, {0x1E95E, 0x1E95F}, {0x1EC71, 0x1ECB4}, - {0x1ED01, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, - {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, - {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, - {0x1EE3B, 0x1EE3B}, {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, - {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, - {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, - {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, - {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, - {0x1EE67, 0x1EE6A}, {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, - {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, - {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, - {0x1EEAB, 0x1EEBB}, {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F02B}, - {0x1F030, 0x1F093}, {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, - {0x1F0C1, 0x1F0CF}, {0x1F0D1, 0x1F0F5}, {0x1F100, 0x1F1AE}, - {0x1F1E6, 0x1F202}, {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, - {0x1F250, 0x1F251}, {0x1F260, 0x1F265}, {0x1F300, 0x1F6D9}, - {0x1F6DC, 0x1F6EC}, {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F7DB}, - {0x1F7E0, 0x1F7EB}, {0x1F7F0, 0x1F80B}, {0x1F810, 0x1F847}, - {0x1F850, 0x1F859}, {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, - {0x1F8B0, 0x1F8BB}, {0x1F8C0, 0x1F8C1}, {0x1F8D0, 0x1F8D8}, - {0x1F900, 0x1FA57}, {0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA7C}, - {0x1FA80, 0x1FAC6}, {0x1FAC8, 0x1FAC8}, {0x1FACC, 0x1FADD}, - {0x1FADF, 0x1FAEB}, {0x1FAEF, 0x1FAFA}, {0x1FB00, 0x1FB92}, - {0x1FB94, 0x1FBFA}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B81E}, - {0x2B820, 0x2CEAD}, {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, - {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x33479}, - {0x3D000, 0x3FC3F}, {0xE0100, 0xE01EF}, + {0x0020, 0x007E}, {0x00A0, 0x0377}, {0x037A, 0x037F}, + {0x0384, 0x038A}, {0x038C, 0x038C}, {0x038E, 0x03A1}, + {0x03A3, 0x052F}, {0x0531, 0x0556}, {0x0558, 0x058F}, + {0x0591, 0x05C9}, {0x05D0, 0x05EA}, {0x05EF, 0x05F4}, + {0x0600, 0x070D}, {0x070F, 0x074A}, {0x074D, 0x07B1}, + {0x07C0, 0x07FA}, {0x07FD, 0x082D}, {0x0830, 0x083E}, + {0x0840, 0x085B}, {0x085E, 0x085E}, {0x0860, 0x086A}, + {0x0870, 0x0891}, {0x0897, 0x0983}, {0x0985, 0x098C}, + {0x098F, 0x0990}, {0x0993, 0x09A8}, {0x09AA, 0x09B0}, + {0x09B2, 0x09B2}, {0x09B6, 0x09B9}, {0x09BC, 0x09C4}, + {0x09C7, 0x09C8}, {0x09CB, 0x09CE}, {0x09D7, 0x09D7}, + {0x09DC, 0x09DD}, {0x09DF, 0x09E3}, {0x09E6, 0x09FE}, + {0x0A01, 0x0A03}, {0x0A05, 0x0A0A}, {0x0A0F, 0x0A10}, + {0x0A13, 0x0A28}, {0x0A2A, 0x0A30}, {0x0A32, 0x0A33}, + {0x0A35, 0x0A36}, {0x0A38, 0x0A39}, {0x0A3C, 0x0A3C}, + {0x0A3E, 0x0A42}, {0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, + {0x0A51, 0x0A51}, {0x0A59, 0x0A5C}, {0x0A5E, 0x0A5E}, + {0x0A66, 0x0A76}, {0x0A81, 0x0A83}, {0x0A85, 0x0A8D}, + {0x0A8F, 0x0A91}, {0x0A93, 0x0AA8}, {0x0AAA, 0x0AB0}, + {0x0AB2, 0x0AB3}, {0x0AB5, 0x0AB9}, {0x0ABC, 0x0AC5}, + {0x0AC7, 0x0AC9}, {0x0ACB, 0x0ACD}, {0x0AD0, 0x0AD0}, + {0x0AE0, 0x0AE3}, {0x0AE6, 0x0AF1}, {0x0AF9, 0x0AFF}, + {0x0B01, 0x0B03}, {0x0B05, 0x0B0C}, {0x0B0F, 0x0B10}, + {0x0B13, 0x0B28}, {0x0B2A, 0x0B30}, {0x0B32, 0x0B33}, + {0x0B35, 0x0B39}, {0x0B3C, 0x0B44}, {0x0B47, 0x0B48}, + {0x0B4B, 0x0B4D}, {0x0B53, 0x0B57}, {0x0B5C, 0x0B5D}, + {0x0B5F, 0x0B63}, {0x0B66, 0x0B77}, {0x0B82, 0x0B83}, + {0x0B85, 0x0B8A}, {0x0B8E, 0x0B90}, {0x0B92, 0x0B95}, + {0x0B99, 0x0B9A}, {0x0B9C, 0x0B9C}, {0x0B9E, 0x0B9F}, + {0x0BA3, 0x0BA4}, {0x0BA8, 0x0BAA}, {0x0BAE, 0x0BB9}, + {0x0BBE, 0x0BC2}, {0x0BC6, 0x0BC8}, {0x0BCA, 0x0BCD}, + {0x0BD0, 0x0BD0}, {0x0BD7, 0x0BD7}, {0x0BE6, 0x0BFA}, + {0x0C00, 0x0C0C}, {0x0C0E, 0x0C10}, {0x0C12, 0x0C28}, + {0x0C2A, 0x0C39}, {0x0C3C, 0x0C44}, {0x0C46, 0x0C48}, + {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56}, {0x0C58, 0x0C5A}, + {0x0C5C, 0x0C5D}, {0x0C60, 0x0C63}, {0x0C66, 0x0C6F}, + {0x0C77, 0x0C8C}, {0x0C8E, 0x0C90}, {0x0C92, 0x0CA8}, + {0x0CAA, 0x0CB3}, {0x0CB5, 0x0CB9}, {0x0CBC, 0x0CC4}, + {0x0CC6, 0x0CC8}, {0x0CCA, 0x0CCD}, {0x0CD5, 0x0CD6}, + {0x0CDC, 0x0CDE}, {0x0CE0, 0x0CE3}, {0x0CE6, 0x0CEF}, + {0x0CF1, 0x0CF3}, {0x0D00, 0x0D0C}, {0x0D0E, 0x0D10}, + {0x0D12, 0x0D44}, {0x0D46, 0x0D48}, {0x0D4A, 0x0D4F}, + {0x0D54, 0x0D63}, {0x0D66, 0x0D7F}, {0x0D81, 0x0D83}, + {0x0D85, 0x0D96}, {0x0D9A, 0x0DB1}, {0x0DB3, 0x0DBB}, + {0x0DBD, 0x0DBD}, {0x0DC0, 0x0DC6}, {0x0DCA, 0x0DCA}, + {0x0DCF, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0DD8, 0x0DDF}, + {0x0DE6, 0x0DEF}, {0x0DF2, 0x0DF4}, {0x0E01, 0x0E3A}, + {0x0E3F, 0x0E5B}, {0x0E81, 0x0E82}, {0x0E84, 0x0E84}, + {0x0E86, 0x0E8A}, {0x0E8C, 0x0EA3}, {0x0EA5, 0x0EA5}, + {0x0EA7, 0x0EBD}, {0x0EC0, 0x0EC4}, {0x0EC6, 0x0EC6}, + {0x0EC8, 0x0ECE}, {0x0ED0, 0x0ED9}, {0x0EDC, 0x0EDF}, + {0x0F00, 0x0F47}, {0x0F49, 0x0F6C}, {0x0F71, 0x0F97}, + {0x0F99, 0x0FBC}, {0x0FBE, 0x0FCC}, {0x0FCE, 0x0FDA}, + {0x1000, 0x10C5}, {0x10C7, 0x10C7}, {0x10CD, 0x10CD}, + {0x10D0, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, + {0x128A, 0x128D}, {0x1290, 0x12B0}, {0x12B2, 0x12B5}, + {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, + {0x12C8, 0x12D6}, {0x12D8, 0x1310}, {0x1312, 0x1315}, + {0x1318, 0x135A}, {0x135D, 0x137C}, {0x1380, 0x1399}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1400, 0x169C}, + {0x16A0, 0x16F8}, {0x1700, 0x1715}, {0x171F, 0x1736}, + {0x1740, 0x1753}, {0x1760, 0x176C}, {0x176E, 0x1770}, + {0x1772, 0x1773}, {0x1780, 0x17DD}, {0x17E0, 0x17E9}, + {0x17F0, 0x17F9}, {0x1800, 0x1819}, {0x1820, 0x1878}, + {0x1880, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, + {0x1920, 0x192B}, {0x1930, 0x193B}, {0x1940, 0x1940}, + {0x1944, 0x196D}, {0x1970, 0x1974}, {0x1980, 0x19AB}, + {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x19DE, 0x1A1B}, + {0x1A1E, 0x1A5E}, {0x1A60, 0x1A7C}, {0x1A7F, 0x1A89}, + {0x1A90, 0x1A99}, {0x1AA0, 0x1AAD}, {0x1AB0, 0x1AF0}, + {0x1B00, 0x1B4C}, {0x1B4E, 0x1BF3}, {0x1BFC, 0x1C37}, + {0x1C3B, 0x1C49}, {0x1C4D, 0x1C8A}, {0x1C90, 0x1CBA}, + {0x1CBD, 0x1CC7}, {0x1CD0, 0x1CFA}, {0x1D00, 0x1F15}, + {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FC4}, {0x1FC6, 0x1FD3}, {0x1FD6, 0x1FDB}, + {0x1FDD, 0x1FEF}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFE}, + {0x2000, 0x2027}, {0x202A, 0x2064}, {0x2066, 0x2071}, + {0x2074, 0x20C4}, {0x20D0, 0x20F0}, {0x2100, 0x218B}, + {0x2190, 0x2429}, {0x2440, 0x244A}, {0x2460, 0x2B73}, + {0x2B76, 0x2CF3}, {0x2CF9, 0x2D25}, {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D70}, + {0x2D7F, 0x2D96}, {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, + {0x2DE0, 0x2E5D}, {0x2E60, 0x2E63}, {0x2E80, 0x2E99}, + {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x303F}, + {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312F}, + {0x3131, 0x318E}, {0x3190, 0x31E5}, {0x31EF, 0x321E}, + {0x3220, 0xA48C}, {0xA490, 0xA4C6}, {0xA4D0, 0xA62B}, + {0xA640, 0xA6F7}, {0xA700, 0xA7DD}, {0xA7E2, 0xA7E2}, + {0xA7F1, 0xA82C}, {0xA830, 0xA839}, {0xA840, 0xA877}, + {0xA880, 0xA8C5}, {0xA8CE, 0xA8D9}, {0xA8E0, 0xA953}, + {0xA95F, 0xA97C}, {0xA980, 0xA9CD}, {0xA9CF, 0xA9D9}, + {0xA9DE, 0xA9FE}, {0xAA00, 0xAA36}, {0xAA40, 0xAA4D}, + {0xAA50, 0xAA59}, {0xAA5C, 0xAAC2}, {0xAADB, 0xAAF6}, + {0xAB01, 0xAB06}, {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, + {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB6D}, + {0xAB70, 0xABED}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, + {0xD7B0, 0xD7C6}, {0xD7CB, 0xD7FB}, {0xE000, 0xFA6D}, + {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, + {0xFB1D, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFDCF}, + {0xFDF0, 0xFE19}, {0xFE20, 0xFE52}, {0xFE54, 0xFE66}, + {0xFE68, 0xFE6B}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, + {0xFEFF, 0xFEFF}, {0xFF01, 0xFFBE}, {0xFFC2, 0xFFC7}, + {0xFFCA, 0xFFCF}, {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, + {0xFFE0, 0xFFE6}, {0xFFE8, 0xFFEE}, {0xFFF9, 0xFFFD}, + {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, + {0x10080, 0x100FA}, {0x10100, 0x10102}, {0x10107, 0x10133}, + {0x10137, 0x1018E}, {0x10190, 0x1019C}, {0x101A0, 0x101A0}, + {0x101D0, 0x101FD}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, + {0x102E0, 0x102FB}, {0x10300, 0x10323}, {0x1032D, 0x1034A}, + {0x10350, 0x1037A}, {0x10380, 0x1039D}, {0x1039F, 0x103C3}, + {0x103C8, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, + {0x104B0, 0x104D3}, {0x104D8, 0x104FB}, {0x10500, 0x10527}, + {0x10530, 0x10563}, {0x1056F, 0x1057A}, {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, {0x105BB, 0x105BC}, + {0x105C0, 0x105F3}, {0x10600, 0x10736}, {0x10740, 0x10755}, + {0x10760, 0x10767}, {0x10780, 0x10785}, {0x10787, 0x107B0}, + {0x107B2, 0x107BF}, {0x10800, 0x10805}, {0x10808, 0x10808}, + {0x1080A, 0x10835}, {0x10837, 0x10838}, {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, {0x10857, 0x1089E}, {0x108A7, 0x108AF}, + {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, + {0x1091F, 0x10939}, {0x1093F, 0x10959}, {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, {0x109D2, 0x10A03}, {0x10A05, 0x10A06}, + {0x10A0C, 0x10A13}, {0x10A15, 0x10A17}, {0x10A19, 0x10A35}, + {0x10A38, 0x10A3A}, {0x10A3F, 0x10A48}, {0x10A50, 0x10A58}, + {0x10A60, 0x10A9F}, {0x10AC0, 0x10AE6}, {0x10AEB, 0x10AF6}, + {0x10B00, 0x10B35}, {0x10B39, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10B99, 0x10B9C}, {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D27}, {0x10D30, 0x10D39}, {0x10D40, 0x10D65}, + {0x10D69, 0x10D85}, {0x10D8E, 0x10D8F}, {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, {0x10EAB, 0x10EAD}, {0x10EB0, 0x10EB1}, + {0x10EC2, 0x10EC7}, {0x10EC9, 0x10EEE}, {0x10EF0, 0x10F27}, + {0x10F30, 0x10F59}, {0x10F70, 0x10F89}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11000, 0x1104D}, {0x11052, 0x11075}, + {0x1107F, 0x110C2}, {0x110CD, 0x110CD}, {0x110D0, 0x110E8}, + {0x110F0, 0x110F9}, {0x11100, 0x11134}, {0x11136, 0x11147}, + {0x11150, 0x11176}, {0x11180, 0x111DF}, {0x111E1, 0x111F4}, + {0x11200, 0x11211}, {0x11213, 0x11241}, {0x11280, 0x11286}, + {0x11288, 0x11288}, {0x1128A, 0x1128D}, {0x1128F, 0x1129D}, + {0x1129F, 0x112A9}, {0x112B0, 0x112EA}, {0x112F0, 0x112F9}, + {0x11300, 0x11303}, {0x11305, 0x1130C}, {0x1130F, 0x11310}, + {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, + {0x11335, 0x11339}, {0x1133B, 0x11344}, {0x11347, 0x11348}, + {0x1134B, 0x1134D}, {0x11350, 0x11350}, {0x11357, 0x11357}, + {0x1135D, 0x11363}, {0x11366, 0x1136C}, {0x11370, 0x11374}, + {0x11380, 0x11389}, {0x1138B, 0x1138B}, {0x1138E, 0x1138E}, + {0x11390, 0x113B5}, {0x113B7, 0x113C0}, {0x113C2, 0x113C2}, + {0x113C5, 0x113C5}, {0x113C7, 0x113CA}, {0x113CC, 0x113D5}, + {0x113D7, 0x113D8}, {0x113E1, 0x113E2}, {0x11400, 0x1145B}, + {0x1145D, 0x11461}, {0x11480, 0x114C7}, {0x114D0, 0x114D9}, + {0x11580, 0x115B5}, {0x115B8, 0x115DD}, {0x11600, 0x11644}, + {0x11650, 0x11659}, {0x11660, 0x1166C}, {0x11680, 0x116B9}, + {0x116C0, 0x116C9}, {0x116D0, 0x116E3}, {0x11700, 0x1171A}, + {0x1171D, 0x1172B}, {0x11730, 0x11746}, {0x11800, 0x1183B}, + {0x118A0, 0x118F2}, {0x118FF, 0x11906}, {0x11909, 0x11909}, + {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x11935}, + {0x11937, 0x11938}, {0x1193B, 0x11946}, {0x11950, 0x11959}, + {0x119A0, 0x119A7}, {0x119AA, 0x119D7}, {0x119DA, 0x119E4}, + {0x11A00, 0x11A47}, {0x11A50, 0x11AA2}, {0x11AB0, 0x11AF8}, + {0x11B00, 0x11B0A}, {0x11B60, 0x11B67}, {0x11BC0, 0x11BE1}, + {0x11BF0, 0x11BF9}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C36}, + {0x11C38, 0x11C45}, {0x11C50, 0x11C6C}, {0x11C70, 0x11C8F}, + {0x11C92, 0x11CA7}, {0x11CA9, 0x11CB6}, {0x11D00, 0x11D06}, + {0x11D08, 0x11D09}, {0x11D0B, 0x11D36}, {0x11D3A, 0x11D3A}, + {0x11D3C, 0x11D3D}, {0x11D3F, 0x11D47}, {0x11D50, 0x11D59}, + {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, {0x11D6A, 0x11D8E}, + {0x11D90, 0x11D91}, {0x11D93, 0x11D98}, {0x11DA0, 0x11DA9}, + {0x11DB0, 0x11DDB}, {0x11DE0, 0x11DE9}, {0x11DF0, 0x11DF1}, + {0x11EE0, 0x11EF8}, {0x11F00, 0x11F10}, {0x11F12, 0x11F3A}, + {0x11F3E, 0x11F5A}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FF1}, + {0x11FFF, 0x12399}, {0x12400, 0x12543}, {0x12550, 0x12686}, + {0x12F90, 0x12FF2}, {0x13000, 0x13455}, {0x13460, 0x143FA}, + {0x14400, 0x14646}, {0x16100, 0x16139}, {0x16800, 0x16A38}, + {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, {0x16A6E, 0x16ABE}, + {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16AF0, 0x16AF5}, + {0x16B00, 0x16B45}, {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, + {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16D40, 0x16D79}, + {0x16E40, 0x16E9A}, {0x16EA0, 0x16EB8}, {0x16EBB, 0x16ED3}, + {0x16F00, 0x16F4A}, {0x16F4F, 0x16F87}, {0x16F8F, 0x16F9F}, + {0x16FE0, 0x16FE4}, {0x16FF0, 0x16FF6}, {0x17000, 0x18CDA}, + {0x18CFF, 0x18D20}, {0x18D80, 0x18DF2}, {0x18E00, 0x19191}, + {0x191A0, 0x191D2}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B128}, {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, {0x1B164, 0x1B168}, + {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, + {0x1BC80, 0x1BC88}, {0x1BC90, 0x1BC99}, {0x1BC9C, 0x1BCA3}, + {0x1CC00, 0x1CCFC}, {0x1CD00, 0x1CEB3}, {0x1CEBA, 0x1CED0}, + {0x1CED2, 0x1CED4}, {0x1CEDD, 0x1CEFD}, {0x1CF00, 0x1CF2D}, + {0x1CF30, 0x1CF46}, {0x1CF50, 0x1CFC3}, {0x1D000, 0x1D0F5}, + {0x1D100, 0x1D245}, {0x1D250, 0x1D281}, {0x1D2C0, 0x1D2D3}, + {0x1D2E0, 0x1D2F3}, {0x1D300, 0x1D356}, {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, + {0x1D540, 0x1D544}, {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A6}, {0x1D6A8, 0x1D7CB}, {0x1D7CE, 0x1DA8B}, + {0x1DA9B, 0x1DA9F}, {0x1DAA1, 0x1DAAF}, {0x1DB00, 0x1DB1C}, + {0x1DF00, 0x1DF81}, {0x1DF90, 0x1DF96}, {0x1DFCD, 0x1E006}, + {0x1E008, 0x1E018}, {0x1E01B, 0x1E021}, {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, {0x1E030, 0x1E06D}, {0x1E08F, 0x1E08F}, + {0x1E100, 0x1E12C}, {0x1E130, 0x1E13D}, {0x1E140, 0x1E149}, + {0x1E14E, 0x1E14F}, {0x1E290, 0x1E2AE}, {0x1E2C0, 0x1E2F9}, + {0x1E2FF, 0x1E2FF}, {0x1E4D0, 0x1E4F9}, {0x1E5D0, 0x1E5FA}, + {0x1E5FF, 0x1E5FF}, {0x1E6C0, 0x1E6DE}, {0x1E6E0, 0x1E6F5}, + {0x1E6FE, 0x1E6FF}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, + {0x1E7ED, 0x1E7EE}, {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, + {0x1E8C7, 0x1E8D6}, {0x1E900, 0x1E94B}, {0x1E950, 0x1E959}, + {0x1E95E, 0x1E95F}, {0x1EC71, 0x1ECB4}, {0x1ED01, 0x1ED3D}, + {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, + {0x1EE24, 0x1EE24}, {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, + {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, + {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, {0x1EE51, 0x1EE52}, + {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, + {0x1EE5B, 0x1EE5B}, {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, + {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, {0x1EE8B, 0x1EE9B}, + {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, + {0x1EEF0, 0x1EEF1}, {0x1F000, 0x1F02B}, {0x1F030, 0x1F093}, + {0x1F0A0, 0x1F0AE}, {0x1F0B1, 0x1F0BF}, {0x1F0C1, 0x1F0CF}, + {0x1F0D1, 0x1F0F5}, {0x1F100, 0x1F1AE}, {0x1F1E6, 0x1F202}, + {0x1F210, 0x1F23B}, {0x1F240, 0x1F248}, {0x1F250, 0x1F251}, + {0x1F260, 0x1F265}, {0x1F300, 0x1F6D9}, {0x1F6DC, 0x1F6EC}, + {0x1F6F0, 0x1F6FC}, {0x1F700, 0x1F7DB}, {0x1F7E0, 0x1F7EB}, + {0x1F7F0, 0x1F80B}, {0x1F810, 0x1F847}, {0x1F850, 0x1F859}, + {0x1F860, 0x1F887}, {0x1F890, 0x1F8AD}, {0x1F8B0, 0x1F8BB}, + {0x1F8C0, 0x1F8C1}, {0x1F8D0, 0x1F8D8}, {0x1F900, 0x1FA57}, + {0x1FA60, 0x1FA6D}, {0x1FA70, 0x1FA7C}, {0x1FA80, 0x1FAC6}, + {0x1FAC8, 0x1FAC8}, {0x1FACC, 0x1FADD}, {0x1FADF, 0x1FAEB}, + {0x1FAEF, 0x1FAFA}, {0x1FB00, 0x1FB92}, {0x1FB94, 0x1FBFA}, + {0x20000, 0x2A6DF}, {0x2A700, 0x2B81E}, {0x2B820, 0x2CEAD}, + {0x2CEB0, 0x2EBE0}, {0x2EBF0, 0x2EE5D}, {0x2F800, 0x2FA1D}, + {0x30000, 0x3134A}, {0x31350, 0x33479}, {0x3D000, 0x3FC3F}, + {0xE0001, 0xE0001}, {0xE0020, 0xE007F}, {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}, }; extern const CharRanges GeneratedPrintableRanges = GeneratedPrintableRangesData; +// General_Category=Cc +static constexpr UnicodeCharRange GeneratedControlCharacterRangesData[] = { + {0x0000, 0x001F}, + {0x007F, 0x009F}, +}; +extern const CharRanges GeneratedControlCharacterRanges = + GeneratedControlCharacterRangesData; + // General_Category=Cf static constexpr UnicodeCharRange GeneratedFormatCharacterRangesData[] = { {0x00AD, 0x00AD}, {0x0600, 0x0605}, {0x061C, 0x061C}, diff --git a/llvm/unittests/Support/UnicodeTest.cpp b/llvm/unittests/Support/UnicodeTest.cpp index d88555512921db..d9ffd827e40ebc 100644 --- a/llvm/unittests/Support/UnicodeTest.cpp +++ b/llvm/unittests/Support/UnicodeTest.cpp @@ -29,13 +29,13 @@ TEST(Unicode, columnWidthUTF8) { EXPECT_EQ(-1, columnWidthUTF8("\x01")); EXPECT_EQ(-1, columnWidthUTF8("\t")); EXPECT_EQ(-1, columnWidthUTF8("aaaaaaaaaa\x01")); - EXPECT_EQ(-1, columnWidthUTF8("\342\200\213")); // 200B ZERO WIDTH SPACE + EXPECT_EQ(0, columnWidthUTF8("\342\200\213")); // 200B ZERO WIDTH SPACE // 00AD SOFT HYPHEN is displayed on most terminals as a space or a dash. Some // text editors display it only when a line is broken at it, some use it as a - // line-break hint, but don't display. We do not take any particular + // line-break hint, but don't display. We choose terminal-oriented // interpretation. - EXPECT_EQ(-1, columnWidthUTF8("\302\255")); + EXPECT_EQ(1, columnWidthUTF8("\302\255")); EXPECT_EQ(0, columnWidthUTF8("\314\200")); // 0300 COMBINING GRAVE ACCENT EXPECT_EQ(1, columnWidthUTF8("\340\270\201")); // 0E01 THAI CHARACTER KO KAI @@ -81,13 +81,14 @@ TEST(Unicode, isPrintable) { EXPECT_FALSE(isPrintable(0x9F)); EXPECT_TRUE(isPrintable(0xAC)); - EXPECT_FALSE(isPrintable(0xAD)); + EXPECT_TRUE(isPrintable(0xAD)); // SOFT HYPHEN is displayed on most terminals + // as either a space or a dash. EXPECT_TRUE(isPrintable(0xAE)); EXPECT_TRUE(isPrintable(0x0377)); // GREEK SMALL LETTER PAMPHYLIAN DIGAMMA EXPECT_FALSE(isPrintable(0x0378)); // <reserved-0378>..<reserved-0379> - EXPECT_FALSE(isPrintable(0x0600)); // ARABIC NUMBER SIGN + EXPECT_TRUE(isPrintable(0x0600)); // ARABIC NUMBER SIGN EXPECT_FALSE(isPrintable(0x1FFFF)); // <reserved-1F774>..<noncharacter-1FFFF> EXPECT_TRUE(isPrintable(0x20000)); // CJK UNIFIED IDEOGRAPH-20000 diff --git a/llvm/unittests/Support/YAMLIOTest.cpp b/llvm/unittests/Support/YAMLIOTest.cpp index f7630f9e93fc8b..8acb37d8298a8a 100644 --- a/llvm/unittests/Support/YAMLIOTest.cpp +++ b/llvm/unittests/Support/YAMLIOTest.cpp @@ -3024,16 +3024,15 @@ TEST(YAMLIO, TestEscaped) { TestEscaped("parameter 'параметр' is unused", "\"parameter 'параметр' is unused\""); - // String with embedded non-printable multibyte UTF-8 sequence (U+00AD soft - // hyphen and U+200B zero-width space). The thing to test here is that we - // emit a unicode-scalar level escape like \uNNNN (at the YAML level), and - // don't just pass the UTF-8 byte sequence through as with quoted printables. - { - const unsigned char foobar[15] = { - 'f', 'o', 'o', 0xC2, 0xAD, // UTF-8 of U+00AD - 'b', 'a', 'r', 0xE2, 0x80, 0x8B, // UTF-8 of U+200B - 'b', 'a', 'z', 0x0}; - TestEscaped((char const *)foobar, "\"foo\\xADbar\\u200Bbaz\""); + // String with embedded non-printable multibyte UTF-8 sequence (U+FFFF + // <not a character>). The thing to test here is that we emit a + // unicode-scalar level escape like \uNNNN (at the YAML level), and don't + // just pass the UTF-8 byte sequence through as with quoted printables. + { + const unsigned char foobar[10] = {'f', 'o', 'o', 0xEF, + 0xBF, 0xBF, // UTF-8 of U+FFFF + 'b', 'a', 'r', 0x0}; + TestEscaped((char const *)foobar, "\"foo\\uFFFFbar\""); } } diff --git a/llvm/utils/UnicodeData/UnicodeCharSetsGenerator.cpp b/llvm/utils/UnicodeData/UnicodeCharSetsGenerator.cpp index fb1918bcddd70c..7c5b1ffdfd92eb 100644 --- a/llvm/utils/UnicodeData/UnicodeCharSetsGenerator.cpp +++ b/llvm/utils/UnicodeData/UnicodeCharSetsGenerator.cpp @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/Support/CommandLine.h" @@ -72,6 +73,7 @@ struct Properties { unsigned MathStart : 1; unsigned MathContinue : 1; unsigned Printable : 1; + unsigned Control : 1; unsigned Format : 1; unsigned Combining : 1; unsigned DoubleWidth : 1; @@ -93,7 +95,8 @@ static StringRef fromXmlChar(const xmlChar *S) { } static bool isPrintableGC(StringRef GC) { - return GC == "Zs" || (!GC.empty() && StringRef("LMNPS").contains(GC.front())); + static SmallSet<StringRef, 5> NonPrintableGCs{"Cc", "Cs", "Zl", "Zp"}; + return !NonPrintableGCs.contains(GC); } // Loads the properties we care about from the flat XML representation of the @@ -157,9 +160,12 @@ static CharacterDatabase loadDatabase(StringRef Path) { P.MathContinue = Value == "Y"; ++Found; } else if (Name == "gc") { - P.Printable = isPrintableGC(Value); - P.Format = Value == "Cf"; - P.Combining = Value == "Mn" || Value == "Me"; + if (Element == "char") { + P.Printable = isPrintableGC(Value); + P.Control = Value == "Cc"; + P.Format = Value == "Cf"; + P.Combining = Value == "Mn" || Value == "Me"; + } ++Found; } else if (Name == "ea") { P.DoubleWidth = Value == "F" || Value == "W"; @@ -243,8 +249,10 @@ int main(int argc, char **argv) { writeFile( LLVMOutput, "llvm::sys::unicode", UCD.Version, [&](raw_ostream &OS) { emitTable(OS, "GeneratedPrintableRanges", - "General_Category=L|M|N|P|S|Zs", Chars, + "General_Category!=Cc|Cs|Zl|Zp", Chars, [](Properties P) { return P.Printable; }); + emitTable(OS, "GeneratedControlCharacterRanges", "General_Category=Cc", + Chars, [](Properties P) { return P.Control; }); emitTable(OS, "GeneratedFormatCharacterRanges", "General_Category=Cf", Chars, [](Properties P) { return P.Format; }); emitTable(OS, "GeneratedCombiningCharacterRanges", >From 7ce81c505b820d1fc48ff51272fc7381a4c0aaa6 Mon Sep 17 00:00:00 2001 From: Harald van Dijk <[email protected]> Date: Wed, 14 Oct 2026 14:52:27 +0100 Subject: [PATCH 4/4] Restore isFormatting check for ForCodepoint case --- clang/lib/Basic/Diagnostic.cpp | 3 ++- clang/test/Lexer/unicode.c | 10 +++++----- clang/test/Preprocessor/ucn-pp-identifier.c | 2 +- clang/unittests/Basic/DiagnosticTest.cpp | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp index 49918b8f90133f..06bf97daea1809 100644 --- a/clang/lib/Basic/Diagnostic.cpp +++ b/clang/lib/Basic/Diagnostic.cpp @@ -1109,7 +1109,8 @@ static void EscapeStringForDiagnostic(StringRef Str, "we must be further along in the string now"); // Do not let through control characters which could mess up the terminal. - if (!llvm::sys::unicode::isControl(CodepointValue)) { + if (!llvm::sys::unicode::isControl(CodepointValue) && + (!ForCodepoint || !llvm::sys::unicode::isFormatting(CodepointValue))) { OutStream << (ForCodepoint ? "'" : "") << StringRef(reinterpret_cast<const char *>(CodepointBegin), std::distance(CodepointBegin, CodepointEnd)) diff --git a/clang/test/Lexer/unicode.c b/clang/test/Lexer/unicode.c index 140c1c23b0db8d..963eca04e0908c 100644 --- a/clang/test/Lexer/unicode.c +++ b/clang/test/Lexer/unicode.c @@ -88,7 +88,7 @@ extern int 👷; // expected-error {{unexpected character '👷' U+1F477}} \ extern int 👷♀; // expected-warning {{declaration does not declare anything}} \ expected-error {{unexpected character '👷' U+1F477}} \ - expected-error {{character '' U+200D not allowed at the start of an identifier}} \ + expected-error {{character U+200D not allowed at the start of an identifier}} \ expected-error {{unexpected character '♀' U+2640}} #else @@ -106,10 +106,10 @@ int *n꞉꞉v = &n;; // expected-warning 2{{treating character '꞉' U+A789 as int v＝［＝］（auto）｛return～x；｝（）; // expected-warning 12{{treating character}} int xx; -// expected-warning@-1 {{identifier contains character '' U+2060 that is invisible in some environments}} -// expected-warning@-2 {{identifier contains character '' U+FEFF that is invisible in some environments}} -// expected-warning@-3 {{identifier contains character '' U+200D that is invisible in some environments}} -int foobar = 0; // expected-warning {{identifier contains character '' U+200B that is invisible in some environments}} +// expected-warning@-1 {{identifier contains character U+2060 that is invisible in some environments}} +// expected-warning@-2 {{identifier contains character U+FEFF that is invisible in some environments}} +// expected-warning@-3 {{identifier contains character U+200D that is invisible in some environments}} +int foobar = 0; // expected-warning {{identifier contains character U+200B that is invisible in some environments}} int x = foobar; // expected-error {{undeclared identifier}} int ∣foo; // expected-error {{unexpected character '∣' U+2223}} diff --git a/clang/test/Preprocessor/ucn-pp-identifier.c b/clang/test/Preprocessor/ucn-pp-identifier.c index 2eeed16f3ced6f..0311d3eb29d1bd 100644 --- a/clang/test/Preprocessor/ucn-pp-identifier.c +++ b/clang/test/Preprocessor/ucn-pp-identifier.c @@ -161,4 +161,4 @@ int a\N{LATIN CAPITAL LETTER A WITH GRAVE??>; #endif // GH64161 -int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character '' U+202D not allowed in an identifier}} +int A\N{LEFT-TO-RIGHT OVERRIDE}; // expected-error {{character U+202D not allowed in an identifier}} diff --git a/clang/unittests/Basic/DiagnosticTest.cpp b/clang/unittests/Basic/DiagnosticTest.cpp index 8ae1cbd2646a00..793431bbbe1548 100644 --- a/clang/unittests/Basic/DiagnosticTest.cpp +++ b/clang/unittests/Basic/DiagnosticTest.cpp @@ -460,7 +460,7 @@ TEST(EscapeSingleCodepointForDiagnosticTest, nonPrintableDisplaysNoQuoted) { EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\n'), "U+000A"); EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\0'), "U+0000"); EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\x1B'), "U+001B"); - EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\uFEFF'), "'' U+FEFF"); + EXPECT_EQ(EscapeSingleCodepointForDiagnostic(U'\uFEFF'), "U+FEFF"); } TEST(EscapeSingleCodepointForDiagnosticTest, nonScalarValues) { _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
