rahil-c commented on code in PR #17632:
URL: https://github.com/apache/hudi/pull/17632#discussion_r2645995893
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceReader.java:
##########
@@ -417,15 +404,55 @@ public void testReadWithMetadataPopulation() throws
Exception {
/**
* Helper method to read all rows from a reader into a list.
- * Makes copies of InternalRows to avoid reuse issues.
+ * Converts UnsafeRow to GenericInternalRow by extracting field values using
type-specific getters.
*/
- private List<InternalRow> readAllRows(HoodieSparkLanceReader reader) throws
IOException {
+ private List<InternalRow> readAllRows(HoodieSparkLanceReader reader,
StructType schema) throws IOException {
List<InternalRow> rows = new ArrayList<>();
- HoodieSchema schema = reader.getSchema();
- try (ClosableIterator<HoodieRecord<InternalRow>> iterator =
reader.getRecordIterator(schema)) {
+ HoodieSchema hoodieSchema = reader.getSchema();
+
+ try (ClosableIterator<HoodieRecord<InternalRow>> iterator =
reader.getRecordIterator(hoodieSchema)) {
while (iterator.hasNext()) {
HoodieRecord<InternalRow> record = iterator.next();
- rows.add(record.getData().copy());
+ InternalRow unsafeRow = record.getData();
+
+ // Extract all field values from UnsafeRow using type-specific getters
+ Object[] values = new Object[schema.fields().length];
+ for (int i = 0; i < schema.fields().length; i++) {
+ if (unsafeRow.isNullAt(i)) {
+ values[i] = null;
+ } else {
+ org.apache.spark.sql.types.DataType dataType =
schema.fields()[i].dataType();
Review Comment:
to fix this
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceReader.java:
##########
@@ -417,15 +404,55 @@ public void testReadWithMetadataPopulation() throws
Exception {
/**
* Helper method to read all rows from a reader into a list.
- * Makes copies of InternalRows to avoid reuse issues.
+ * Converts UnsafeRow to GenericInternalRow by extracting field values using
type-specific getters.
*/
- private List<InternalRow> readAllRows(HoodieSparkLanceReader reader) throws
IOException {
+ private List<InternalRow> readAllRows(HoodieSparkLanceReader reader,
StructType schema) throws IOException {
List<InternalRow> rows = new ArrayList<>();
- HoodieSchema schema = reader.getSchema();
- try (ClosableIterator<HoodieRecord<InternalRow>> iterator =
reader.getRecordIterator(schema)) {
+ HoodieSchema hoodieSchema = reader.getSchema();
+
+ try (ClosableIterator<HoodieRecord<InternalRow>> iterator =
reader.getRecordIterator(hoodieSchema)) {
while (iterator.hasNext()) {
HoodieRecord<InternalRow> record = iterator.next();
- rows.add(record.getData().copy());
+ InternalRow unsafeRow = record.getData();
+
+ // Extract all field values from UnsafeRow using type-specific getters
+ Object[] values = new Object[schema.fields().length];
+ for (int i = 0; i < schema.fields().length; i++) {
+ if (unsafeRow.isNullAt(i)) {
+ values[i] = null;
+ } else {
+ org.apache.spark.sql.types.DataType dataType =
schema.fields()[i].dataType();
Review Comment:
to fix this to not use full path
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
