Hi Paul, On Sun, Jun 16, 2019 at 9:50 PM Paul Gevers <elb...@debian.org> wrote: > On 16-06-2019 11:20, László Böszörményi (GCS) wrote: > > The debdiff is larger for the following changes. The backported > > security fixes are no longer under debian/patches but inline. The ABI > > break, called the 'ICU-20250' issue upstream is reversed with a patch. > > Then the s/63.1/63.2/ changes, etc. > > Can you please provide a diff between the patches-applied tree of the > current buster version and a patches-applied tree of the current sid > version? Of course, attached. The diff size went down from 165 kB to 39 kB as you see, even if the documentation and s/63.1/63.2/ changes are still in as well.
Regards, Laszlo/GCS
diff -Nur icu-63.1/readme.html icu-63.2/readme.html --- icu-63.1/readme.html 2018-10-15 18:02:37.000000000 +0000 +++ icu-63.2/readme.html 2019-04-11 22:38:30.000000000 +0000 @@ -3,7 +3,7 @@ <html lang="en-US" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US"> <head> - <title>ReadMe for ICU 63.1</title> + <title>ReadMe for ICU 63.2</title> <meta name="COPYRIGHT" content= "Copyright (C) 2016 and later: Unicode, Inc. and others. License & terms of use: http://www.unicode.org/copyright.html"/> <!-- meta name="COPYRIGHT" content= @@ -32,7 +32,7 @@ International Components for Unicode<br/> <span class="only-rc">Release Candidate</span> <span class="only-milestone">(Milestone Release)</span> - <abbr title="International Components for Unicode">ICU</abbr> 63.1 ReadMe + <abbr title="International Components for Unicode">ICU</abbr> 63.2 ReadMe </h1> <!-- Shouldn't need to comment/uncomment this paragraph, just change the body class --> diff -Nur icu-63.1/source/common/umutablecptrie.cpp icu-63.2/source/common/umutablecptrie.cpp --- icu-63.1/source/common/umutablecptrie.cpp 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/common/umutablecptrie.cpp 2019-06-16 20:23:58.000000000 +0000 @@ -60,6 +60,7 @@ constexpr int32_t INDEX_3_18BIT_BLOCK_LENGTH = UCPTRIE_INDEX_3_BLOCK_LENGTH + UCPTRIE_INDEX_3_BLOCK_LENGTH / 8; class AllSameBlocks; +class MixedBlocks; class MutableCodePointTrie : public UMemory { public: @@ -92,8 +93,10 @@ void maskValues(uint32_t mask); UChar32 findHighStart() const; int32_t compactWholeDataBlocks(int32_t fastILimit, AllSameBlocks &allSameBlocks); - int32_t compactData(int32_t fastILimit, uint32_t *newData, int32_t dataNullIndex); - int32_t compactIndex(int32_t fastILimit, UErrorCode &errorCode); + int32_t compactData( + int32_t fastILimit, uint32_t *newData, int32_t newDataCapacity, + int32_t dataNullIndex, MixedBlocks &mixedBlocks, UErrorCode &errorCode); + int32_t compactIndex(int32_t fastILimit, MixedBlocks &mixedBlocks, UErrorCode &errorCode); int32_t compactTrie(int32_t fastILimit, UErrorCode &errorCode); uint32_t *index = nullptr; @@ -548,28 +551,8 @@ } } -inline bool -equalBlocks(const uint32_t *s, const uint32_t *t, int32_t length) { - while (length > 0 && *s == *t) { - ++s; - ++t; - --length; - } - return length == 0; -} - -inline bool -equalBlocks(const uint16_t *s, const uint32_t *t, int32_t length) { - while (length > 0 && *s == *t) { - ++s; - ++t; - --length; - } - return length == 0; -} - -inline bool -equalBlocks(const uint16_t *s, const uint16_t *t, int32_t length) { +template<typename UIntA, typename UIntB> +bool equalBlocks(const UIntA *s, const UIntB *t, int32_t length) { while (length > 0 && *s == *t) { ++s; ++t; @@ -585,36 +568,6 @@ } /** Search for an identical block. */ -int32_t findSameBlock(const uint32_t *p, int32_t pStart, int32_t length, - const uint32_t *q, int32_t qStart, int32_t blockLength) { - // Ensure that we do not even partially get past length. - length -= blockLength; - - q += qStart; - while (pStart <= length) { - if (equalBlocks(p + pStart, q, blockLength)) { - return pStart; - } - ++pStart; - } - return -1; -} - -int32_t findSameBlock(const uint16_t *p, int32_t pStart, int32_t length, - const uint32_t *q, int32_t qStart, int32_t blockLength) { - // Ensure that we do not even partially get past length. - length -= blockLength; - - q += qStart; - while (pStart <= length) { - if (equalBlocks(p + pStart, q, blockLength)) { - return pStart; - } - ++pStart; - } - return -1; -} - int32_t findSameBlock(const uint16_t *p, int32_t pStart, int32_t length, const uint16_t *q, int32_t qStart, int32_t blockLength) { // Ensure that we do not even partially get past length. @@ -655,30 +608,9 @@ * Look for maximum overlap of the beginning of the other block * with the previous, adjacent block. */ -int32_t getOverlap(const uint32_t *p, int32_t length, - const uint32_t *q, int32_t qStart, int32_t blockLength) { - int32_t overlap = blockLength - 1; - U_ASSERT(overlap <= length); - q += qStart; - while (overlap > 0 && !equalBlocks(p + (length - overlap), q, overlap)) { - --overlap; - } - return overlap; -} - -int32_t getOverlap(const uint16_t *p, int32_t length, - const uint32_t *q, int32_t qStart, int32_t blockLength) { - int32_t overlap = blockLength - 1; - U_ASSERT(overlap <= length); - q += qStart; - while (overlap > 0 && !equalBlocks(p + (length - overlap), q, overlap)) { - --overlap; - } - return overlap; -} - -int32_t getOverlap(const uint16_t *p, int32_t length, - const uint16_t *q, int32_t qStart, int32_t blockLength) { +template<typename UIntA, typename UIntB> +int32_t getOverlap(const UIntA *p, int32_t length, + const UIntB *q, int32_t qStart, int32_t blockLength) { int32_t overlap = blockLength - 1; U_ASSERT(overlap <= length); q += qStart; @@ -807,6 +739,171 @@ int32_t refCounts[CAPACITY]; }; +// Custom hash table for mixed-value blocks to be found anywhere in the +// compacted data or index so far. +class MixedBlocks { +public: + MixedBlocks() {} + ~MixedBlocks() { + uprv_free(table); + } + + bool init(int32_t maxLength, int32_t newBlockLength) { + // We store actual data indexes + 1 to reserve 0 for empty entries. + int32_t maxDataIndex = maxLength - newBlockLength + 1; + int32_t newLength; + if (maxDataIndex <= 0xfff) { // 4k + newLength = 6007; + shift = 12; + mask = 0xfff; + } else if (maxDataIndex <= 0x7fff) { // 32k + newLength = 50021; + shift = 15; + mask = 0x7fff; + } else if (maxDataIndex <= 0x1ffff) { // 128k + newLength = 200003; + shift = 17; + mask = 0x1ffff; + } else { + // maxDataIndex up to around MAX_DATA_LENGTH, ca. 1.1M + newLength = 1500007; + shift = 21; + mask = 0x1fffff; + } + if (newLength > capacity) { + uprv_free(table); + table = (uint32_t *)uprv_malloc(newLength * 4); + if (table == nullptr) { + return false; + } + capacity = newLength; + } + length = newLength; + uprv_memset(table, 0, length * 4); + + blockLength = newBlockLength; + return true; + } + + template<typename UInt> + void extend(const UInt *data, int32_t minStart, int32_t prevDataLength, int32_t newDataLength) { + int32_t start = prevDataLength - blockLength; + if (start >= minStart) { + ++start; // Skip the last block that we added last time. + } else { + start = minStart; // Begin with the first full block. + } + for (int32_t end = newDataLength - blockLength; start <= end; ++start) { + uint32_t hashCode = makeHashCode(data, start); + addEntry(data, start, hashCode, start); + } + } + + template<typename UIntA, typename UIntB> + int32_t findBlock(const UIntA *data, const UIntB *blockData, int32_t blockStart) const { + uint32_t hashCode = makeHashCode(blockData, blockStart); + int32_t entryIndex = findEntry(data, blockData, blockStart, hashCode); + if (entryIndex >= 0) { + return (table[entryIndex] & mask) - 1; + } else { + return -1; + } + } + + int32_t findAllSameBlock(const uint32_t *data, uint32_t blockValue) const { + uint32_t hashCode = makeHashCode(blockValue); + int32_t entryIndex = findEntry(data, blockValue, hashCode); + if (entryIndex >= 0) { + return (table[entryIndex] & mask) - 1; + } else { + return -1; + } + } + +private: + template<typename UInt> + uint32_t makeHashCode(const UInt *blockData, int32_t blockStart) const { + int32_t blockLimit = blockStart + blockLength; + uint32_t hashCode = blockData[blockStart++]; + do { + hashCode = 37 * hashCode + blockData[blockStart++]; + } while (blockStart < blockLimit); + return hashCode; + } + + uint32_t makeHashCode(uint32_t blockValue) const { + uint32_t hashCode = blockValue; + for (int32_t i = 1; i < blockLength; ++i) { + hashCode = 37 * hashCode + blockValue; + } + return hashCode; + } + + template<typename UInt> + void addEntry(const UInt *data, int32_t blockStart, uint32_t hashCode, int32_t dataIndex) { + U_ASSERT(0 <= dataIndex && dataIndex < (int32_t)mask); + int32_t entryIndex = findEntry(data, data, blockStart, hashCode); + if (entryIndex < 0) { + table[~entryIndex] = (hashCode << shift) | (dataIndex + 1); + } + } + + template<typename UIntA, typename UIntB> + int32_t findEntry(const UIntA *data, const UIntB *blockData, int32_t blockStart, + uint32_t hashCode) const { + uint32_t shiftedHashCode = hashCode << shift; + int32_t initialEntryIndex = (hashCode % (length - 1)) + 1; // 1..length-1 + for (int32_t entryIndex = initialEntryIndex;;) { + uint32_t entry = table[entryIndex]; + if (entry == 0) { + return ~entryIndex; + } + if ((entry & ~mask) == shiftedHashCode) { + int32_t dataIndex = (entry & mask) - 1; + if (equalBlocks(data + dataIndex, blockData + blockStart, blockLength)) { + return entryIndex; + } + } + entryIndex = nextIndex(initialEntryIndex, entryIndex); + } + } + + int32_t findEntry(const uint32_t *data, uint32_t blockValue, uint32_t hashCode) const { + uint32_t shiftedHashCode = hashCode << shift; + int32_t initialEntryIndex = (hashCode % (length - 1)) + 1; // 1..length-1 + for (int32_t entryIndex = initialEntryIndex;;) { + uint32_t entry = table[entryIndex]; + if (entry == 0) { + return ~entryIndex; + } + if ((entry & ~mask) == shiftedHashCode) { + int32_t dataIndex = (entry & mask) - 1; + if (allValuesSameAs(data + dataIndex, blockLength, blockValue)) { + return entryIndex; + } + } + entryIndex = nextIndex(initialEntryIndex, entryIndex); + } + } + + inline int32_t nextIndex(int32_t initialEntryIndex, int32_t entryIndex) const { + // U_ASSERT(0 < initialEntryIndex && initialEntryIndex < length); + return (entryIndex + initialEntryIndex) % length; + } + + // Hash table. + // The length is a prime number, larger than the maximum data length. + // The "shift" lower bits store a data index + 1. + // The remaining upper bits store a partial hashCode of the block data values. + uint32_t *table = nullptr; + int32_t capacity = 0; + int32_t length = 0; + int32_t shift = 0; + uint32_t mask = 0; + + int32_t blockLength = 0; +}; + int32_t MutableCodePointTrie::compactWholeDataBlocks(int32_t fastILimit, AllSameBlocks &allSameBlocks) { #ifdef UCPTRIE_DEBUG bool overflow = false; @@ -962,8 +1059,9 @@ * * It does not try to find an optimal order of writing, deduplicating, and overlapping blocks. */ -int32_t MutableCodePointTrie::compactData(int32_t fastILimit, - uint32_t *newData, int32_t dataNullIndex) { +int32_t MutableCodePointTrie::compactData( + int32_t fastILimit, uint32_t *newData, int32_t newDataCapacity, + int32_t dataNullIndex, MixedBlocks &mixedBlocks, UErrorCode &errorCode) { #ifdef UCPTRIE_DEBUG int32_t countSame=0, sumOverlaps=0; bool printData = dataLength == 29088 /* line.brk */ || @@ -983,8 +1081,14 @@ #endif } - int32_t iLimit = highStart >> UCPTRIE_SHIFT_3; int32_t blockLength = UCPTRIE_FAST_DATA_BLOCK_LENGTH; + if (!mixedBlocks.init(newDataCapacity, blockLength)) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + mixedBlocks.extend(newData, 0, 0, newDataLength); + + int32_t iLimit = highStart >> UCPTRIE_SHIFT_3; int32_t inc = SMALL_DATA_BLOCKS_PER_BMP_BLOCK; int32_t fastLength = 0; for (int32_t i = ASCII_I_LIMIT; i < iLimit; i += inc) { @@ -992,12 +1096,17 @@ blockLength = UCPTRIE_SMALL_DATA_BLOCK_LENGTH; inc = 1; fastLength = newDataLength; + if (!mixedBlocks.init(newDataCapacity, blockLength)) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + mixedBlocks.extend(newData, 0, 0, newDataLength); } if (flags[i] == ALL_SAME) { uint32_t value = index[i]; - int32_t n; // Find an earlier part of the data array of length blockLength // that is filled with this value. + int32_t n = mixedBlocks.findAllSameBlock(newData, value); // If we find a match, and the current block is the data null block, // and it is not a fast block but matches the start of a fast block, // then we need to continue looking. @@ -1005,12 +1114,10 @@ // and not all of the rest of the fast block is filled with this value. // Otherwise trie.getRange() would detect that the fast block starts at // dataNullOffset and assume incorrectly that it is filled with the null value. - for (int32_t start = 0; - (n = findAllSameBlock(newData, start, newDataLength, - value, blockLength)) >= 0 && - i == dataNullIndex && i >= fastILimit && n < fastLength && - isStartOfSomeFastBlock(n, index, fastILimit); - start = n + 1) {} + while (n >= 0 && i == dataNullIndex && i >= fastILimit && n < fastLength && + isStartOfSomeFastBlock(n, index, fastILimit)) { + n = findAllSameBlock(newData, n + 1, newDataLength, value, blockLength); + } if (n >= 0) { DEBUG_DO(++countSame); index[i] = n; @@ -1023,14 +1130,16 @@ } #endif index[i] = newDataLength - n; + int32_t prevDataLength = newDataLength; while (n < blockLength) { newData[newDataLength++] = value; ++n; } + mixedBlocks.extend(newData, 0, prevDataLength, newDataLength); } } else if (flags[i] == MIXED) { const uint32_t *block = data + index[i]; - int32_t n = findSameBlock(newData, 0, newDataLength, block, 0, blockLength); + int32_t n = mixedBlocks.findBlock(newData, block, 0); if (n >= 0) { DEBUG_DO(++countSame); index[i] = n; @@ -1043,9 +1152,11 @@ } #endif index[i] = newDataLength - n; + int32_t prevDataLength = newDataLength; while (n < blockLength) { newData[newDataLength++] = block[n++]; } + mixedBlocks.extend(newData, 0, prevDataLength, newDataLength); } } else /* SAME_AS */ { uint32_t j = index[i]; @@ -1061,7 +1172,8 @@ return newDataLength; } -int32_t MutableCodePointTrie::compactIndex(int32_t fastILimit, UErrorCode &errorCode) { +int32_t MutableCodePointTrie::compactIndex(int32_t fastILimit, MixedBlocks &mixedBlocks, + UErrorCode &errorCode) { int32_t fastIndexLength = fastILimit >> (UCPTRIE_FAST_SHIFT - UCPTRIE_SHIFT_3); if ((highStart >> UCPTRIE_FAST_SHIFT) <= fastIndexLength) { // Only the linear fast index, no multi-stage index tables. @@ -1095,6 +1207,12 @@ } } + if (!mixedBlocks.init(fastIndexLength, UCPTRIE_INDEX_3_BLOCK_LENGTH)) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + mixedBlocks.extend(fastIndex, 0, 0, fastIndexLength); + // Examine index-3 blocks. For each determine one of: // - same as the index-3 null block // - same as a fast-index block @@ -1105,6 +1223,7 @@ // Also determine an upper limit for the index-3 table length. int32_t index3Capacity = 0; i3FirstNull = index3NullOffset; + bool hasLongI3Blocks = false; // If the fast index covers the whole BMP, then // the multi-stage index is only for supplementary code points. // Otherwise, the multi-stage index covers all of Unicode. @@ -1129,13 +1248,13 @@ index3Capacity += UCPTRIE_INDEX_3_BLOCK_LENGTH; } else { index3Capacity += INDEX_3_18BIT_BLOCK_LENGTH; + hasLongI3Blocks = true; } i3FirstNull = 0; } } else { if (oredI3 <= 0xffff) { - int32_t n = findSameBlock(fastIndex, 0, fastIndexLength, - index, i, UCPTRIE_INDEX_3_BLOCK_LENGTH); + int32_t n = mixedBlocks.findBlock(fastIndex, index, i); if (n >= 0) { flags[i] = I3_BMP; index[i] = n; @@ -1146,6 +1265,7 @@ } else { flags[i] = I3_18; index3Capacity += INDEX_3_18BIT_BLOCK_LENGTH; + hasLongI3Blocks = true; } } i = j; @@ -1166,6 +1286,18 @@ } uprv_memcpy(index16, fastIndex, fastIndexLength * 2); + if (!mixedBlocks.init(index16Capacity, UCPTRIE_INDEX_3_BLOCK_LENGTH)) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + MixedBlocks longI3Blocks; + if (hasLongI3Blocks) { + if (!longI3Blocks.init(index16Capacity, INDEX_3_18BIT_BLOCK_LENGTH)) { + errorCode = U_MEMORY_ALLOCATION_ERROR; + return 0; + } + } + // Compact the index-3 table and write an uncompacted version of the index-2 table. uint16_t index2[UNICODE_LIMIT >> UCPTRIE_SHIFT_2]; // index2Capacity int32_t i2Length = 0; @@ -1185,8 +1317,7 @@ } else if (f == I3_BMP) { i3 = index[i]; } else if (f == I3_16) { - int32_t n = findSameBlock(index16, index3Start, indexLength, - index, i, UCPTRIE_INDEX_3_BLOCK_LENGTH); + int32_t n = mixedBlocks.findBlock(index16, index, i); if (n >= 0) { i3 = n; } else { @@ -1198,12 +1329,18 @@ index, i, UCPTRIE_INDEX_3_BLOCK_LENGTH); } i3 = indexLength - n; + int32_t prevIndexLength = indexLength; while (n < UCPTRIE_INDEX_3_BLOCK_LENGTH) { index16[indexLength++] = index[i + n++]; } + mixedBlocks.extend(index16, index3Start, prevIndexLength, indexLength); + if (hasLongI3Blocks) { + longI3Blocks.extend(index16, index3Start, prevIndexLength, indexLength); + } } } else { U_ASSERT(f == I3_18); + U_ASSERT(hasLongI3Blocks); // Encode an index-3 block that contains one or more data indexes exceeding 16 bits. int32_t j = i; int32_t jLimit = i + UCPTRIE_INDEX_3_BLOCK_LENGTH; @@ -1236,8 +1373,7 @@ index16[k++] = v; index16[k - 9] = upperBits; } while (j < jLimit); - int32_t n = findSameBlock(index16, index3Start, indexLength, - index16, indexLength, INDEX_3_18BIT_BLOCK_LENGTH); + int32_t n = longI3Blocks.findBlock(index16, index16, indexLength); if (n >= 0) { i3 = n | 0x8000; } else { @@ -1249,6 +1385,7 @@ index16, indexLength, INDEX_3_18BIT_BLOCK_LENGTH); } i3 = (indexLength - n) | 0x8000; + int32_t prevIndexLength = indexLength; if (n > 0) { int32_t start = indexLength; while (n < INDEX_3_18BIT_BLOCK_LENGTH) { @@ -1257,6 +1394,10 @@ } else { indexLength += INDEX_3_18BIT_BLOCK_LENGTH; } + mixedBlocks.extend(index16, index3Start, prevIndexLength, indexLength); + if (hasLongI3Blocks) { + longI3Blocks.extend(index16, index3Start, prevIndexLength, indexLength); + } } } if (index3NullOffset < 0 && i3FirstNull >= 0) { @@ -1279,16 +1420,23 @@ } // Compact the index-2 table and write the index-1 table. + static_assert(UCPTRIE_INDEX_2_BLOCK_LENGTH == UCPTRIE_INDEX_3_BLOCK_LENGTH, + "must re-init mixedBlocks"); int32_t blockLength = UCPTRIE_INDEX_2_BLOCK_LENGTH; int32_t i1 = fastIndexLength; for (int32_t i = 0; i < i2Length; i += blockLength) { - if ((i2Length - i) < blockLength) { + int32_t n; + if ((i2Length - i) >= blockLength) { + // normal block + U_ASSERT(blockLength == UCPTRIE_INDEX_2_BLOCK_LENGTH); + n = mixedBlocks.findBlock(index16, index2, i); + } else { // highStart is inside the last index-2 block. Shorten it. blockLength = i2Length - i; + n = findSameBlock(index16, index3Start, indexLength, + index2, i, blockLength); } int32_t i2; - int32_t n = findSameBlock(index16, index3Start, indexLength, - index2, i, blockLength); if (n >= 0) { i2 = n; } else { @@ -1299,9 +1447,11 @@ n = getOverlap(index16, indexLength, index2, i, blockLength); } i2 = indexLength - n; + int32_t prevIndexLength = indexLength; while (n < blockLength) { index16[indexLength++] = index2[i + n++]; } + mixedBlocks.extend(index16, index3Start, prevIndexLength, indexLength); } // Set the index-1 table entry. index16[i1++] = i2; @@ -1369,7 +1519,11 @@ uprv_memcpy(newData, asciiData, sizeof(asciiData)); int32_t dataNullIndex = allSameBlocks.findMostUsed(); - int32_t newDataLength = compactData(fastILimit, newData, dataNullIndex); + + MixedBlocks mixedBlocks; + int32_t newDataLength = compactData(fastILimit, newData, newDataCapacity, + dataNullIndex, mixedBlocks, errorCode); + if (U_FAILURE(errorCode)) { return 0; } U_ASSERT(newDataLength <= newDataCapacity); uprv_free(data); data = newData; @@ -1394,7 +1548,7 @@ dataNullOffset = UCPTRIE_NO_DATA_NULL_OFFSET; } - int32_t indexLength = compactIndex(fastILimit, errorCode); + int32_t indexLength = compactIndex(fastILimit, mixedBlocks, errorCode); highStart = realHighStart; return indexLength; } diff -Nur icu-63.1/source/common/umutex.h icu-63.2/source/common/umutex.h --- icu-63.1/source/common/umutex.h 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/common/umutex.h 2019-04-11 22:38:30.000000000 +0000 @@ -54,15 +54,23 @@ #include <atomic> -U_NAMESPACE_BEGIN - // Export an explicit template instantiation of std::atomic<int32_t>. // When building DLLs for Windows this is required as it is used as a data member of the exported SharedObject class. // See digitlst.h, pluralaffix.h, datefmt.h, and others for similar examples. -#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN +#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN && !defined(U_IN_DOXYGEN) + #if defined(__clang__) + // Suppress the warning that the explicit instantiation after explicit specialization has no effect. + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Winstantiation-after-specialization" + #endif template struct U_COMMON_API std::atomic<int32_t>; + #if defined(__clang__) + #pragma clang diagnostic pop + #endif #endif +U_NAMESPACE_BEGIN + typedef std::atomic<int32_t> u_atomic_int32_t; #define ATOMIC_INT32_T_INITIALIZER(val) ATOMIC_VAR_INIT(val) diff -Nur icu-63.1/source/common/unicode/uvernum.h icu-63.2/source/common/unicode/uvernum.h --- icu-63.1/source/common/unicode/uvernum.h 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/common/unicode/uvernum.h 2019-04-11 22:38:30.000000000 +0000 @@ -66,7 +66,7 @@ * This value will change in the subsequent releases of ICU * @stable ICU 2.6 */ -#define U_ICU_VERSION_MINOR_NUM 1 +#define U_ICU_VERSION_MINOR_NUM 2 /** The current ICU patchlevel version as an integer. * This value will change in the subsequent releases of ICU @@ -121,7 +121,7 @@ * This value will change in the subsequent releases of ICU * @stable ICU 2.4 */ -#define U_ICU_VERSION "63.1" +#define U_ICU_VERSION "63.2" /** * The current ICU library major version number as a string, for library name suffixes. @@ -140,7 +140,7 @@ /** Data version in ICU4C. * @internal ICU 4.4 Internal Use Only **/ -#define U_ICU_DATA_VERSION "63.1" +#define U_ICU_DATA_VERSION "63.2" #endif /* U_HIDE_INTERNAL_API */ /*=========================================================================== diff -Nur icu-63.1/source/configure icu-63.2/source/configure --- icu-63.1/source/configure 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/configure 2019-04-11 22:38:30.000000000 +0000 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for ICU 63.1. +# Generated by GNU Autoconf 2.69 for ICU 63.2. # # Report bugs to <http://icu-project.org/bugs>. # @@ -582,8 +582,8 @@ # Identity of this package. PACKAGE_NAME='ICU' PACKAGE_TARNAME='International Components for Unicode' -PACKAGE_VERSION='63.1' -PACKAGE_STRING='ICU 63.1' +PACKAGE_VERSION='63.2' +PACKAGE_STRING='ICU 63.2' PACKAGE_BUGREPORT='http://icu-project.org/bugs' PACKAGE_URL='http://icu-project.org' @@ -1370,7 +1370,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures ICU 63.1 to adapt to many kinds of systems. +\`configure' configures ICU 63.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1437,7 +1437,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ICU 63.1:";; + short | recursive ) echo "Configuration of ICU 63.2:";; esac cat <<\_ACEOF @@ -1574,7 +1574,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ICU configure 63.1 +ICU configure 63.2 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2266,7 +2266,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ICU $as_me 63.1, which was +It was created by ICU $as_me 63.2, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -8434,7 +8434,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ICU $as_me 63.1, which was +This file was extended by ICU $as_me 63.2, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -8488,7 +8488,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -ICU config.status 63.1 +ICU config.status 63.2 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Binary files icu-63.1/source/data/in/icudt63l.dat and icu-63.2/source/data/in/icudt63l.dat differ diff -Nur icu-63.1/source/i18n/japancal.cpp icu-63.2/source/i18n/japancal.cpp --- icu-63.1/source/i18n/japancal.cpp 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/i18n/japancal.cpp 2019-04-11 22:38:30.000000000 +0000 @@ -18,6 +18,16 @@ #if !UCONFIG_NO_FORMATTING #if U_PLATFORM_HAS_WINUWP_API == 0 #include <stdlib.h> // getenv() is not available in UWP env +#else +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +# define VC_EXTRALEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +#include <windows.h> #endif #include "cmemory.h" #include "erarules.h" diff -Nur icu-63.1/source/i18n/unicode/numberrangeformatter.h icu-63.2/source/i18n/unicode/numberrangeformatter.h --- icu-63.1/source/i18n/unicode/numberrangeformatter.h 2018-10-15 18:02:37.000000000 +0000 +++ icu-63.2/source/i18n/unicode/numberrangeformatter.h 2019-04-11 22:38:30.000000000 +0000 @@ -185,8 +185,14 @@ * Export an explicit template instantiation. See datefmt.h * (When building DLLs for Windows this is required.) */ -#if U_PF_WINDOWS <= U_PLATFORM && U_PLATFORM <= U_PF_CYGWIN && !defined(U_IN_DOXYGEN) -template struct U_I18N_API std::atomic<impl::NumberRangeFormatterImpl*>; +#if U_PLATFORM == U_PF_WINDOWS && !defined(U_IN_DOXYGEN) +} // namespace icu::number +U_NAMESPACE_END + +template struct U_I18N_API std::atomic< U_NAMESPACE_QUALIFIER number::impl::NumberRangeFormatterImpl*>; + +U_NAMESPACE_BEGIN +namespace number { // icu::number #endif /** \endcond */ diff -Nur icu-63.1/source/test/intltest/incaltst.cpp icu-63.2/source/test/intltest/incaltst.cpp --- icu-63.1/source/test/intltest/incaltst.cpp 2018-09-29 00:34:42.000000000 +0000 +++ icu-63.2/source/test/intltest/incaltst.cpp 2019-04-11 22:38:30.000000000 +0000 @@ -77,6 +77,7 @@ CASE(7,TestPersian); CASE(8,TestPersianFormat); CASE(9,TestTaiwan); + CASE(10,TestJapaneseHeiseiToReiwa); default: name = ""; break; } } @@ -626,23 +627,23 @@ // Test simple parse/format with adopt UDate aDate = 0; - // Test parse with missing era (should default to current era, heisei) + // Test parse with missing era (should default to current era) // Test parse with incomplete information logln("Testing parse w/ missing era..."); - SimpleDateFormat *fmt = new SimpleDateFormat(UnicodeString("y.M.d"), Locale("ja_JP@calendar=japanese"), status); + SimpleDateFormat *fmt = new SimpleDateFormat(UnicodeString("y/M/d"), Locale("ja_JP@calendar=japanese"), status); CHECK(status, "creating date format instance"); if(!fmt) { errln("Couldn't create en_US instance"); } else { UErrorCode s2 = U_ZERO_ERROR; cal2->clear(); - UnicodeString samplestr("1.1.9"); + UnicodeString samplestr("1/5/9"); logln(UnicodeString() + "Test Year: " + samplestr); aDate = fmt->parse(samplestr, s2); ParsePosition pp=0; fmt->parse(samplestr, *cal2, pp); - CHECK(s2, "parsing the 1.1.9 string"); - logln("*cal2 after 119 parse:"); + CHECK(s2, "parsing the 1/5/9 string"); + logln("*cal2 after 159 parse:"); str.remove(); fmt2->format(aDate, str); logln(UnicodeString() + "as Gregorian Calendar: " + str); @@ -653,7 +654,7 @@ int32_t expectYear = 1; int32_t expectEra = JapaneseCalendar::getCurrentEra(); if((gotYear!=1) || (gotEra != expectEra)) { - errln(UnicodeString("parse "+samplestr+" of 'y.m.d' as Japanese Calendar, expected year ") + expectYear + + errln(UnicodeString("parse "+samplestr+" of 'y/m/d' as Japanese Calendar, expected year ") + expectYear + UnicodeString(" and era ") + expectEra +", but got year " + gotYear + " and era " + gotEra + " (Gregorian:" + str +")"); } else { logln(UnicodeString() + " year: " + gotYear + ", era: " + gotEra); @@ -666,7 +667,7 @@ // Test simple parse/format with adopt UDate aDate = 0; - // Test parse with missing era (should default to current era, heisei) + // Test parse with missing era (should default to current era) // Test parse with incomplete information logln("Testing parse w/ just year..."); SimpleDateFormat *fmt = new SimpleDateFormat(UnicodeString("y"), Locale("ja_JP@calendar=japanese"), status); @@ -678,7 +679,7 @@ cal2->clear(); UnicodeString samplestr("1"); logln(UnicodeString() + "Test Year: " + samplestr); - aDate = fmt->parse(samplestr, s2); + aDate = fmt->parse(samplestr, s2); // Should be parsed as the first day of the current era ParsePosition pp=0; fmt->parse(samplestr, *cal2, pp); CHECK(s2, "parsing the 1 string"); @@ -691,7 +692,7 @@ int32_t gotYear = cal2->get(UCAL_YEAR, s2); int32_t gotEra = cal2->get(UCAL_ERA, s2); int32_t expectYear = 1; - int32_t expectEra = 235; //JapaneseCalendar::kCurrentEra; + int32_t expectEra = JapaneseCalendar::getCurrentEra(); if((gotYear!=1) || (gotEra != expectEra)) { errln(UnicodeString("parse "+samplestr+" of 'y' as Japanese Calendar, expected year ") + expectYear + UnicodeString(" and era ") + expectEra +", but got year " + gotYear + " and era " + gotEra + " (Gregorian:" + str +")"); @@ -707,6 +708,40 @@ delete fmt2; } +void IntlCalendarTest::TestJapaneseHeiseiToReiwa() { + Calendar *cal; + UErrorCode status = U_ZERO_ERROR; + cal = Calendar::createInstance(status); + CHECK(status, UnicodeString("Creating default Gregorian Calendar")); + cal->set(2019, UCAL_APRIL, 29); + + DateFormat *jfmt = DateFormat::createDateInstance(DateFormat::LONG, "ja@calendar=japanese"); + CHECK(status, UnicodeString("Creating date format ja@calendar=japanese")) + + const char* EXPECTED_FORMAT[4] = { + "\\u5E73\\u621031\\u5E744\\u670829\\u65E5", // Heisei 31 April 29 + "\\u5E73\\u621031\\u5E744\\u670830\\u65E5", // Heisei 31 April 30 + "\\u4EE4\\u548c1\\u5E745\\u67081\\u65E5", // Reiwa 1 May 1 + "\\u4EE4\\u548c1\\u5E745\\u67082\\u65E5" // Reiwa 1 May 2 + }; + + for (int32_t i = 0; i < 4; i++) { + UnicodeString dateStr; + UDate d = cal->getTime(status); + CHECK(status, UnicodeString("Get test date")); + jfmt->format(d, dateStr); + UnicodeString expected(UnicodeString(EXPECTED_FORMAT[i], -1, US_INV).unescape()); + if (expected.compare(dateStr) != 0) { + errln(UnicodeString("Formatting year:") + cal->get(UCAL_YEAR, status) + " month:" + + cal->get(UCAL_MONTH, status) + " day:" + (cal->get(UCAL_DATE, status) + 1) + + " - expected: " + expected + " / actual: " + dateStr); + } + cal->add(UCAL_DATE, 1, status); + CHECK(status, UnicodeString("Add 1 day")); + } + delete jfmt; + delete cal; +} diff -Nur icu-63.1/source/test/intltest/incaltst.h icu-63.2/source/test/intltest/incaltst.h --- icu-63.1/source/test/intltest/incaltst.h 2018-09-29 00:34:42.000000000 +0000 +++ icu-63.2/source/test/intltest/incaltst.h 2019-04-11 22:38:30.000000000 +0000 @@ -34,6 +34,7 @@ void TestJapanese(void); void TestJapaneseFormat(void); void TestJapanese3860(void); + void TestJapaneseHeiseiToReiwa(void); void TestPersian(void); void TestPersianFormat(void); diff -Nur icu-63.1/source/test/intltest/numbertest.h icu-63.2/source/test/intltest/numbertest.h --- icu-63.1/source/test/intltest/numbertest.h 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/test/intltest/numbertest.h 2019-04-11 22:38:30.000000000 +0000 @@ -10,6 +10,7 @@ #include "intltest.h" #include "number_affixutils.h" #include "numparse_stringsegment.h" +#include "numrange_impl.h" #include "unicode/locid.h" #include "unicode/numberformatter.h" #include "unicode/numberrangeformatter.h" diff -Nur icu-63.1/source/test/testdata/format.txt icu-63.2/source/test/testdata/format.txt --- icu-63.1/source/test/testdata/format.txt 2018-10-01 22:39:56.000000000 +0000 +++ icu-63.2/source/test/testdata/format.txt 2019-04-11 22:38:30.000000000 +0000 @@ -488,42 +488,44 @@ "AD 02008" }, - // Japanese - { - "en_US@calendar=japanese", - "", - "PATTERN=G y", - "YEAR=8", - "Heisei 8" - }, - { - "en_US@calendar=japanese", - "", - "PATTERN=G yy", - "YEAR=8", - "Heisei 08" - }, - { - "en_US@calendar=japanese", - "", - "PATTERN=G yyy", - "YEAR=8", - "Heisei 008" - }, - { - "en_US@calendar=japanese", - "", - "PATTERN=G yyyy", - "YEAR=8", - "Heisei 0008" - }, - { - "en_US@calendar=japanese", - "", - "PATTERN=G yyyyy", - "YEAR=8", - "Heisei 00008" - }, +// The following test case is commented out as the current era +// depends on the current time when the test is run. +// // Japanese +// { +// "en_US@calendar=japanese", +// "", +// "PATTERN=G y", +// "YEAR=8", +// "Reiwa 8" +// }, +// { +// "en_US@calendar=japanese", +// "", +// "PATTERN=G yy", +// "YEAR=8", +// "Reiwa 08" +// }, +// { +// "en_US@calendar=japanese", +// "", +// "PATTERN=G yyy", +// "YEAR=8", +// "Reiwa 008" +// }, +// { +// "en_US@calendar=japanese", +// "", +// "PATTERN=G yyyy", +// "YEAR=8", +// "Reiwa 0008" +// }, +// { +// "en_US@calendar=japanese", +// "", +// "PATTERN=G yyyyy", +// "YEAR=8", +// "Reiwa 00008" +// }, } }