uros-db commented on code in PR #47771:
URL: https://github.com/apache/spark/pull/47771#discussion_r1725038872
##########
common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java:
##########
@@ -550,6 +558,197 @@ public static UTF8String toTitleCase(final UTF8String
target, final int collatio
BreakIterator.getWordInstance(locale)));
}
+ /**
+ * This 'HashMap' is introduced as a performance speedup. Since titlecasing
a codepoint can result
+ * in more than a single codepoint, for correctness, we would use
'UCharacter.toTitleCase(String)'
+ * which returns a 'String'. If we use 'UCharacter.toTitleCase(int)' (the
version of the same
+ * function which converts a single codepoint to its titlecase codepoint),
it would be faster than
+ * the previously mentioned version, but the problem here is that we don't
handle when titlecasing
+ * a codepoint yields more than 1 codepoint. Since there are only 48
codepoints that are mapped to
+ * more than 1 codepoint when titlecased, they are precalculated here, so
that the faster function
+ * for titlecasing could be used in combination with this 'HashMap' in the
method
+ * 'appendCodepointToTitleCase'.
+ */
+ private static final HashMap<Integer, String>
codepointOneToManyTitleCasePrecalculation =
+ new HashMap<>(){{
+ put(223, "Ss");
+ put(329, "ʼN");
+ put(496, "J̌");
+ put(912, "Ϊ́");
+ put(944, "Ϋ́");
+ put(1415, "Եւ");
+ put(7830, "H̱");
+ put(7831, "T̈");
+ put(7832, "W̊");
+ put(7833, "Y̊");
+ put(7834, "Aʾ");
+ put(8016, "Υ̓");
+ put(8018, "Υ̓̀");
+ put(8020, "Υ̓́");
+ put(8022, "Υ̓͂");
+ put(8114, "Ὰͅ");
+ put(8116, "Άͅ");
+ put(8118, "Α͂");
+ put(8119, "ᾼ͂");
+ put(8130, "Ὴͅ");
+ put(8132, "Ήͅ");
+ put(8134, "Η͂");
+ put(8135, "ῌ͂");
+ put(8146, "Ϊ̀");
+ put(8147, "Ϊ́");
+ put(8150, "Ι͂");
+ put(8151, "Ϊ͂");
+ put(8162, "Ϋ̀");
+ put(8163, "Ϋ́");
+ put(8164, "Ρ̓");
+ put(8166, "Υ͂");
+ put(8167, "Ϋ͂");
+ put(8178, "Ὼͅ");
+ put(8180, "Ώͅ");
+ put(8182, "Ω͂");
+ put(8183, "ῼ͂");
+ put(64256, "Ff");
+ put(64257, "Fi");
+ put(64258, "Fl");
+ put(64259, "Ffi");
+ put(64260, "Ffl");
+ put(64261, "St");
+ put(64262, "St");
+ put(64275, "Մն");
+ put(64276, "Մե");
+ put(64277, "Մի");
+ put(64278, "Վն");
+ put(64279, "Մխ");
+ }};
+
+ /**
+ * Title casing a string according to a new behaviour. Iterates over the
string and title cases
+ * the first character in each word, and lowercases every other character.
Handles lowercasing
+ * capital Greek letter sigma ('Σ') separately, taking into account if it
should be a small final
+ * Greek sigma ('ς') or small non-final Greek sigma ('σ'). Words are
separated by ASCII
+ * space(\u0020).
+ *
+ * @param target UTF8String to be title cased
+ * @return title cased target
+ */
+ public static UTF8String toTitleCaseICU(UTF8String target) {
+
+ // In the default UTF8String implementation, `toLowerCase` method
implicitly does UTF8String
+ // validation (replacing invalid UTF-8 byte sequences with Unicode
replacement character
+ // U+FFFD), but now we have to do the validation manually.
+ target = target.makeValid();
+
+ // Building the title cased target with 'sb'.
+ StringBuilder sb = new StringBuilder();
+
+ // 'newWord' is true if the current character is the beginning of a word,
false otherwise.
+ boolean newWord = true;
+ // We are maintaining if the current character is preceded by a cased
letter.
+ // This is used when lowercasing capital Greek letter sigma ('Σ'), to
figure out if it should be
+ // lowercased into σ or ς.
+ boolean precededByCasedLetter = false;
+
+ // 'offset' is a byte offset in target's byte array pointing to the
beginning of the character
+ // that we need to process next.
+ int offset = 0;
+ int len = target.numBytes();
+
+ while(offset < len) {
+ // We will actually call 'codePointFrom()' 2 times for each character in
the worst case (once
+ // here, and once in 'followedByCasedLetter'). Example of a string where
we call it 2 times
+ // for almost every character is 'ΣΣΣΣΣ' (a string consisting only of
Greek capital sigma)
+ // and 'Σ`````' (a string consisting of a Greek capital sigma, followed
by case-ignorable
+ // characters).
+ int codepoint = target.codePointFrom(offset);
+ // Appending the correctly cased character onto 'sb'.
+ appendTitleCasedCodepoint(sb, codepoint, newWord, precededByCasedLetter,
target, offset);
+ // Updating 'newWord', 'precededByCasedLetter' and 'offset' to be ready
for the next character
+ // that we will process.
+ newWord = (codepoint == ASCII_SPACE_CODEPOINT);
+ if(!UCharacter.hasBinaryProperty(codepoint, UProperty.CASE_IGNORABLE)) {
+ precededByCasedLetter = UCharacter.hasBinaryProperty(codepoint,
UProperty.CASED);
+ }
+ offset += UTF8String.numBytesForFirstByte(target.getByte(offset));
+ }
+ return UTF8String.fromString(sb.toString());
Review Comment:
we may be able to use `UTF8StringBuilder` here, provided that we extend it
with an `appendCodePoint` method
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
