uros-db commented on code in PR #47771:
URL: https://github.com/apache/spark/pull/47771#discussion_r1718686867
##########
common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java:
##########
@@ -1267,6 +1268,128 @@ public static UTF8String[] icuSplitSQL(final UTF8String
string, final UTF8String
return strings.toArray(new UTF8String[0]);
}
+ /**
+ * Title casing a string according to a new behaviour. Iterates over the
string and title cases
+ * the first character in each word, and lowercases every other character.
Handles lowercasing
+ * capital Greek letter sigma ('Σ') separately, taking into account if it
should be a small final
+ * Greek sigma ('ς') or small non-final Greek sigma ('σ'). Words are
separated by ASCII
+ * space(\u0020).
+ *
+ * @param target UTF8String to be title cased
+ * @return title cased target
+ */
+ public static UTF8String toTitleCaseICU(UTF8String target) {
+
+ Iterator<Integer> codepointIterator = target.codePointIterator(
+ CodePointIteratorType.CODE_POINT_ITERATOR_MAKE_VALID);
+
+ // Building the title cased target with 'sb'.
+ StringBuilder sb = new StringBuilder();
+ // 'newWord' is true if the current character is the beginning of a word,
false otherwise.
+ boolean newWord = true;
+ // We are maintaining if the current character is preceded by a cased
letter.
+ // This is used when lowercasing capital Greek letter sigma ('Σ'), to
figure out if it should be
+ // lowercased into σ or ς.
+ boolean precededByCasedLetter = false;
+
+ // 'offset' is a byte offset in target's byte array pointing to the
beginning of the character
+ // that we need to process next(this is only actually used in
appendLowerCasedGreekCapitalSigma)
+ int offset = 0;
+
+ while(codepointIterator.hasNext()) {
+ int codepoint = codepointIterator.next();
+ // Appending the correctly cased character onto 'sb'.
+ appendTitleCasedCodepoint(sb, codepoint, newWord, precededByCasedLetter,
target, offset);
+ // Updating 'newWord', 'precededByCasedLetter' and 'offset' to be ready
for the next character
+ // that we will process.
+ newWord = (codepoint == ASCII_SPACE_CODEPOINT);
+ if(!UCharacter.hasBinaryProperty(codepoint, UProperty.CASE_IGNORABLE)) {
+ precededByCasedLetter = UCharacter.hasBinaryProperty(codepoint,
UProperty.CASED);
+ }
+ offset += UTF8String.numBytesForFirstByte(target.getByte(offset));
+ }
+ return UTF8String.fromString(sb.toString());
+ }
+
+ private static void appendTitleCasedCodepoint(
+ StringBuilder sb,
+ int codepoint,
+ boolean isAfterAsciiSpace,
+ boolean precededByCasedLetter,
+ UTF8String target,
+ int offset) {
+ if(isAfterAsciiSpace) {
+ // Titlecasing a character if it is in the beginning of a new word.
+ sb.append(codepointToTitleString(codepoint));
+ return;
+ }
+ if(codepoint == CAPITAL_SIGMA) {
+ // Handling capital Greek letter sigma ('Σ').
+ appendLowerCasedGreekCapitalSigma(sb, precededByCasedLetter, target,
offset);
+ return;
+ }
+ // If it's not the beginning of a word, or a capital Greek letter sigma
('Σ'), we lowercase the
+ // character.
+ sb.append(
+ toLowerCase(UTF8String.fromString(new
String(Character.toChars(codepoint)))).toString());
+ }
+
+ private static void appendLowerCasedGreekCapitalSigma(
+ StringBuilder sb,
+ boolean precededByCasedLetter,
+ UTF8String target,
+ int offset) {
+ int codepoint;
+ if (!followedByCasedLetter(target,offset) && precededByCasedLetter) {
+ codepoint = SMALL_FINAL_SIGMA;
+ }
+ else {
+ codepoint = SMALL_NON_FINAL_SIGMA;
+ }
+ sb.appendCodePoint(codepoint);
+ }
+
+ /**
+ * Checks if the character beginning at 'offset'(in 'targets' byte array) is
followed by a cased
+ * letter.
+ *
+ * @param target
+ * @param offset
+ * @return
+ */
+ private static boolean followedByCasedLetter(UTF8String target, int offset) {
+ // Moving the offset one character forward, so we could start the linear
search from there.
+ offset += UTF8String.numBytesForFirstByte(target.getByte(offset));
+ int len = target.numBytes();
+
+ while(offset < len) {
+ int codepoint = target.codePointFrom(offset);
+
+ if(UCharacter.hasBinaryProperty(codepoint, UProperty.CASE_IGNORABLE)) {
+ offset += UTF8String.numBytesForFirstByte(target.getByte(offset));
+ continue;
+ }
+ return UCharacter.hasBinaryProperty(codepoint, UProperty.CASED);
+ }
+ return false;
+ }
+
+ /**
+ * Titlecases a single character using the ICU root locale rules.
+ *
+ * @param codepoint is a character that is needed to be title cased
+ * @return a java String(whose length can be more than 1 character) which
corresponds to the title
+ * case of the codepoint
+ */
+ private static String codepointToTitleString(int codepoint) {
+ return UCharacter.toTitleCase(new
String(Character.toChars(codepoint)),null);
+ }
+
+ private static final int ASCII_SPACE_CODEPOINT = 32;
+ private static final int CAPITAL_SIGMA = 0x03A3;
+ private static final int SMALL_NON_FINAL_SIGMA = 0x03C3;
+ private static final int SMALL_FINAL_SIGMA = 0x03C2;
Review Comment:
also, I believe that we've already hard-coded these two values somewhere in
this file (getLowercaseCodePoint, and possibly somewhere else too), so let's
make sure to update everything to use the newly added constants
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
