yihua commented on a change in pull request #5004:
URL: https://github.com/apache/hudi/pull/5004#discussion_r832453975
##########
File path:
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
##########
@@ -160,6 +187,89 @@ public void testWriteReadHFile(boolean populateMetaFields,
boolean testAvroWithM
}
}
+ @Override
+ @Test
+ public void testWriteReadWithEvolvedSchema() throws Exception {
+ // Disable the test with evolved schema for HFile since it's not supported
+ }
+
+ @Test
+ public void testReadHFileFormatRecords() throws Exception {
+ writeFileWithSimpleSchema();
+ FileSystem fs = FSUtils.getFs(getFilePath().toString(), new
Configuration());
+ byte[] content = FileIOUtils.readAsByteArray(
+ fs.open(getFilePath()), (int)
fs.getFileStatus(getFilePath()).getLen());
+ // Reading byte array in HFile format, without actual file path
+ HoodieHFileReader<GenericRecord> hfileReader =
+ new HoodieHFileReader<>(fs, new Path(DUMMY_BASE_PATH), content);
+ Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
+ assertEquals(NUM_RECORDS, hfileReader.getTotalRecords());
+ verifySimpleRecords(hfileReader.getRecordIterator(avroSchema));
+ }
+
+ @Test
+ public void testReaderGetRecordIterator() throws Exception {
+ writeFileWithSimpleSchema();
+ HoodieHFileReader<GenericRecord> hfileReader =
+ (HoodieHFileReader<GenericRecord>) createReader(new Configuration());
+ List<String> keys =
+ IntStream.concat(IntStream.range(40, NUM_RECORDS * 2),
IntStream.range(10, 20))
+ .mapToObj(i -> "key" + String.format("%02d",
i)).collect(Collectors.toList());
+ Schema avroSchema =
getSchemaFromResource(TestHoodieReaderWriterBase.class, "/exampleSchema.avsc");
+ Iterator<GenericRecord> iterator = hfileReader.getRecordIterator(keys,
avroSchema);
+
+ List<Integer> expectedIds =
+ IntStream.concat(IntStream.range(40, NUM_RECORDS), IntStream.range(10,
20))
+ .boxed().collect(Collectors.toList());
+ int index = 0;
+ while (iterator.hasNext()) {
+ GenericRecord record = iterator.next();
+ String key = "key" + String.format("%02d", expectedIds.get(index));
+ assertEquals(key, record.get("_row_key").toString());
+ assertEquals(Integer.toString(expectedIds.get(index)),
record.get("time").toString());
+ assertEquals(expectedIds.get(index), record.get("number"));
+ index++;
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "/hudi_0_9_hbase_1_2_3", "/hudi_0_10_hbase_1_2_3",
"/hudi_0_11_hbase_2_4_9"})
+ public void testHoodieHFileCompatibility(String hfilePrefix) throws
IOException {
Review comment:
Makes sense. The fixtures for older releases should not be changed
given they are generated from public releases and should be used as is for
compatibility tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
