mkaravel commented on code in PR #46732:
URL: https://github.com/apache/spark/pull/46732#discussion_r1630682407
##########
common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java:
##########
@@ -594,10 +594,40 @@ public void testInitCap() throws SparkException {
assertInitCap("aB 世 de", "UNICODE_CI", "Ab 世 De");
assertInitCap("ÄBĆΔE", "UNICODE_CI", "Äbćδe");
// Case-variable character length
- assertInitCap("İo", "UTF8_BINARY", "İo");
- assertInitCap("İo", "UTF8_BINARY_LCASE", "İo");
- assertInitCap("İo", "UNICODE", "İo");
- assertInitCap("İo", "UNICODE_CI", "İo");
+ assertInitCap("İo", "UTF8_BINARY", "I\u0307o");
+ assertInitCap("İo", "UTF8_BINARY_LCASE", "İo");
+ assertInitCap("İo", "UNICODE", "I\u0307o");
+ assertInitCap("İo", "UNICODE_CI", "İo");
+ assertInitCap("i\u0307o", "UTF8_BINARY", "I\u0307o");
+ assertInitCap("i\u0307o", "UTF8_BINARY_LCASE", "I\u0307o");
+ assertInitCap("i\u0307o", "UNICODE", "I\u0307o");
+ assertInitCap("i\u0307o", "UNICODE_CI", "I\u0307o");
+ // Different possible word boundaries
+ assertInitCap("a b c", "UTF8_BINARY", "A B C");
+ assertInitCap("a b c", "UNICODE", "A B C");
+ assertInitCap("a b c", "UTF8_BINARY_LCASE", "A B C");
+ assertInitCap("a b c", "UNICODE_CI", "A B C");
+ assertInitCap("a.b,c", "UTF8_BINARY", "A.b,c");
+ assertInitCap("a.b,c", "UNICODE", "A.b,c");
+ assertInitCap("a.b,c", "UTF8_BINARY_LCASE", "A.b,C");
+ assertInitCap("a.b,c", "UNICODE_CI", "A.b,C");
+ assertInitCap("a. b-c", "UTF8_BINARY", "A. B-c");
+ assertInitCap("a. b-c", "UNICODE", "A. B-c");
+ assertInitCap("a. b-c", "UTF8_BINARY_LCASE", "A. B-C");
+ assertInitCap("a. b-c", "UNICODE_CI", "A. B-C");
+ assertInitCap("a?b世c", "UTF8_BINARY", "A?b世c");
+ assertInitCap("a?b世c", "UNICODE", "A?b世c");
+ assertInitCap("a?b世c", "UTF8_BINARY_LCASE", "A?B世C");
+ assertInitCap("a?b世c", "UNICODE_CI", "A?B世C");
+ // Titlecase characters that are different from uppercase characters
+ assertInitCap("dzDZDz", "UTF8_BINARY", "Dzdzdz");
+ assertInitCap("dzDZDz", "UNICODE", "Dzdzdz");
+ assertInitCap("dzDZDz", "UTF8_BINARY_LCASE", "Dzdzdz");
+ assertInitCap("dzDZDz", "UNICODE_CI", "Dzdzdz");
+ assertInitCap("džaba Ljubav NJegova", "UTF8_BINARY", "Džaba Ljubav Njegova");
+ assertInitCap("džaba Ljubav NJegova", "UNICODE", "Džaba Ljubav Njegova");
+ assertInitCap("džaba Ljubav NJegova", "UTF8_BINARY_LCASE", "Džaba Ljubav
Njegova");
+ assertInitCap("džaba Ljubav NJegova", "UNICODE_CI", "Džaba Ljubav Njegova");
Review Comment:
I would suggest adding a few more test cases involving ligatures and
conditional case mapping:
```scala
assertInitCap("ß fi ffi ff st ΣΗΜΕΡΙΝΟΣ ΑΣΗΜΕΝΙΟΣ İOTA", "UTF8_BINARY_LCASE", "Ss
Fi Ffi Ff St Σημερινος Ασημενιος İota");
assertInitCap("ß fi ffi ff st ΣΗΜΕΡΙΝΟΣ ΑΣΗΜΕΝΙΟΣ İOTA", "UNICODE", "Ss Fi Ffi Ff
St Σημερινος Ασημενιος İota");
assertInitCap("ß fi ffi ff st ΣΗΜΕΡΙΝΟΣ ΑΣΗΜΕΝΙΟΣ İOTA", "UNICODE_CI", "Ss Fi Ffi
Ff St Σημερινος Ασημενιος İota");
```
Notice that I on purpose did not include the UTF8_BINARY collation above.
The current implementation for InitCap is highly broken. The corresponding test
case (which I suggest to add anyways to keep track of) is:
```scala
assertInitCap("ß fi ffi ff st ΣΗΜΕΡΙΝΟΣ ΑΣΗΜΕΝΙΟΣ İOTA", "UTF8_BINARY", "ß fi ffi ff
st Σημερινος Ασημενιος I\u0307ota");
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
