the-other-tim-brown commented on code in PR #13383:
URL: https://github.com/apache/hudi/pull/13383#discussion_r2127183896
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java:
##########
@@ -137,16 +136,16 @@ public static String[] getExpressionIndexColumnNames() {
};
}
- public static List<Row> getRowsWithExpressionIndexMetadata(List<Row>
rowsForFilePath, String partition, String filePath, long fileSize) {
- return rowsForFilePath.stream().map(row -> {
- scala.collection.immutable.Seq<Object> indexMetadata =
JavaScalaConverters.convertJavaListToScalaList(Arrays.asList(partition,
filePath, fileSize));
+ public static ClosableIterator<Row>
getRowsWithExpressionIndexMetadata(ClosableIterator<InternalRow>
rowsForFilePath, SparkRowSerDe sparkRowSerDe, String partition, String
filePath, long fileSize) {
+ return new CloseableMappingIterator<>(rowsForFilePath, row -> {
+ Seq<Object> indexMetadata =
JavaScalaConverters.convertJavaListToScalaList(Arrays.asList(partition,
filePath, fileSize));
Row expressionIndexRow = Row.fromSeq(indexMetadata);
List<Row> rows = new ArrayList<>(2);
- rows.add(row);
+ rows.add(sparkRowSerDe.deserializeRow(row));
rows.add(expressionIndexRow);
- scala.collection.immutable.Seq<Row> rowSeq =
JavaScalaConverters.convertJavaListToScalaList(rows);
+ Seq<Row> rowSeq = JavaScalaConverters.convertJavaListToScalaList(rows);
Review Comment:
that does not work for me, are you able to get that compiling locally? could
be my setup
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
