MaxGekk commented on code in PR #46880:
URL: https://github.com/apache/spark/pull/46880#discussion_r1755351285
##########
sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java:
##########
@@ -272,4 +277,42 @@ private static byte[] aesInternal(
throw QueryExecutionErrors.aesCryptoError(e.getMessage());
}
}
+
+ public static ArrayData getSentences(
+ UTF8String str,
+ UTF8String language,
+ UTF8String country) {
+ if (str == null) return null;
+ Locale locale;
+ if (language != null && country != null) {
+ locale = new Locale(language.toString(), country.toString());
+ } else if (language != null) {
+ locale = new Locale(language.toString());
+ } else {
+ locale = Locale.US;
+ }
+ String sentences = str.toString();
+ BreakIterator sentenceInstance = BreakIterator.getSentenceInstance(locale);
+ sentenceInstance.setText(sentences);
+
+ int sentenceIndex = 0;
+ List<GenericArrayData> res = new ArrayList<>();
+ while (sentenceInstance.next() != BreakIterator.DONE) {
+ String sentence = sentences.substring(sentenceIndex,
sentenceInstance.current());
+ sentenceIndex = sentenceInstance.current();
+ BreakIterator wordInstance = BreakIterator.getWordInstance(locale);
+ wordInstance.setText(sentence);
+ int wordIndex = 0;
+ List<UTF8String> words = new ArrayList<>();
+ while (wordInstance.next() != BreakIterator.DONE) {
+ String word = sentence.substring(wordIndex, wordInstance.current());
+ wordIndex = wordInstance.current();
+ if (Character.isLetterOrDigit(word.charAt(0))) {
+ words.add(UTF8String.fromString(word));
+ }
+ }
+ res.add(new GenericArrayData(words.toArray(new UTF8String[0])));
Review Comment:
nit:
```suggestion
res.add(new GenericArrayData(words.toArray()));
```
##########
sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionImplUtils.java:
##########
@@ -272,4 +277,42 @@ private static byte[] aesInternal(
throw QueryExecutionErrors.aesCryptoError(e.getMessage());
}
}
+
+ public static ArrayData getSentences(
+ UTF8String str,
+ UTF8String language,
+ UTF8String country) {
+ if (str == null) return null;
+ Locale locale;
+ if (language != null && country != null) {
+ locale = new Locale(language.toString(), country.toString());
+ } else if (language != null) {
+ locale = new Locale(language.toString());
+ } else {
+ locale = Locale.US;
+ }
+ String sentences = str.toString();
+ BreakIterator sentenceInstance = BreakIterator.getSentenceInstance(locale);
+ sentenceInstance.setText(sentences);
+
+ int sentenceIndex = 0;
+ List<GenericArrayData> res = new ArrayList<>();
+ while (sentenceInstance.next() != BreakIterator.DONE) {
+ String sentence = sentences.substring(sentenceIndex,
sentenceInstance.current());
+ sentenceIndex = sentenceInstance.current();
+ BreakIterator wordInstance = BreakIterator.getWordInstance(locale);
+ wordInstance.setText(sentence);
+ int wordIndex = 0;
+ List<UTF8String> words = new ArrayList<>();
+ while (wordInstance.next() != BreakIterator.DONE) {
+ String word = sentence.substring(wordIndex, wordInstance.current());
+ wordIndex = wordInstance.current();
+ if (Character.isLetterOrDigit(word.charAt(0))) {
+ words.add(UTF8String.fromString(word));
+ }
+ }
+ res.add(new GenericArrayData(words.toArray(new UTF8String[0])));
+ }
+ return new GenericArrayData(res.toArray(new GenericArrayData[0]));
Review Comment:
nit:
```suggestion
return new GenericArrayData(res.toArray());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
