This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new be0c30735b8 [MINOR] Move hoodie hfile/orc reader/writer test cases
from hudi-client-common to hudi-common (#9103)
be0c30735b8 is described below
commit be0c30735b8c19338249dfb936c7da40adeff18c
Author: Mulavar <[email protected]>
AuthorDate: Sun Sep 17 12:59:25 2023 +0800
[MINOR] Move hoodie hfile/orc reader/writer test cases from
hudi-client-common to hudi-common (#9103)
Co-authored-by: Y Ethan Guo <[email protected]>
---
.../hudi/io/storage/TestHoodieHFileReaderWriter.java | 15 ++++++---------
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 2 +-
.../hudi/io/storage/TestHoodieReaderWriterBase.java | 0
.../src/test/resources/exampleEvolvedSchema.avsc | 0
.../test/resources/exampleEvolvedSchemaChangeOrder.avsc | 0
.../resources/exampleEvolvedSchemaColumnRequire.avsc | 0
.../test/resources/exampleEvolvedSchemaColumnType.avsc | 0
.../test/resources/exampleEvolvedSchemaDeleteColumn.avsc | 0
.../src/test/resources/exampleSchema.avsc | 0
.../src/test/resources/exampleSchemaWithMetaFields.avsc | 0
.../src/test/resources/exampleSchemaWithUDT.avsc | 0
...udi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile | Bin
.../test/resources/hudi_0_10_hbase_1_2_3_complex.hfile | Bin
.../test/resources/hudi_0_10_hbase_1_2_3_simple.hfile | Bin
...udi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile | Bin
.../test/resources/hudi_0_11_hbase_2_4_9_complex.hfile | Bin
.../test/resources/hudi_0_11_hbase_2_4_9_simple.hfile | Bin
...hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile | Bin
.../test/resources/hudi_0_9_hbase_1_2_3_complex.hfile | Bin
.../src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile | Bin
20 files changed, 7 insertions(+), 10 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
similarity index 97%
rename from
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
rename to
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
index af4de5b771e..a7de5fe396b 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
+++
b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
@@ -19,16 +19,16 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex;
+import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
@@ -57,6 +57,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.Spliterator;
import java.util.Spliterators;
@@ -95,20 +96,16 @@ public class TestHoodieHFileReaderWriter extends
TestHoodieReaderWriterBase {
protected HoodieAvroHFileWriter createWriter(
Schema avroSchema, boolean populateMetaFields) throws Exception {
String instantTime = "000";
- HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
- .withPath(DUMMY_BASE_PATH)
- .withIndexConfig(HoodieIndexConfig.newBuilder()
- .bloomFilterNumEntries(1000).bloomFilterFPP(0.00001).build())
- .withPopulateMetaFields(populateMetaFields)
- .build();
Configuration conf = new Configuration();
+ Properties props = new Properties();
+ props.setProperty(HoodieTableConfig.POPULATE_META_FIELDS.key(),
Boolean.toString(populateMetaFields));
TaskContextSupplier mockTaskContextSupplier =
Mockito.mock(TaskContextSupplier.class);
Supplier<Integer> partitionSupplier = Mockito.mock(Supplier.class);
when(mockTaskContextSupplier.getPartitionIdSupplier()).thenReturn(partitionSupplier);
when(partitionSupplier.get()).thenReturn(10);
return (HoodieAvroHFileWriter)HoodieFileWriterFactory.getFileWriter(
- instantTime, getFilePath(), conf, writeConfig.getStorageConfig(),
avroSchema, mockTaskContextSupplier,
writeConfig.getRecordMerger().getRecordType());
+ instantTime, getFilePath(), conf,
HoodieStorageConfig.newBuilder().fromProperties(props).build(), avroSchema,
mockTaskContextSupplier, HoodieRecord.HoodieRecordType.AVRO);
}
@Override
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
rename to
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
index 438024d2f26..98614be25c3 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
+++
b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
@@ -22,8 +22,8 @@ import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
-import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.config.HoodieStorageConfig;
+import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
import org.apache.avro.Schema;
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
b/hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
rename to
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterBase.java
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchema.avsc
b/hudi-common/src/test/resources/exampleEvolvedSchema.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchema.avsc
rename to hudi-common/src/test/resources/exampleEvolvedSchema.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc
b/hudi-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc
rename to hudi-common/src/test/resources/exampleEvolvedSchemaChangeOrder.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc
b/hudi-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc
rename to hudi-common/src/test/resources/exampleEvolvedSchemaColumnRequire.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc
b/hudi-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc
rename to hudi-common/src/test/resources/exampleEvolvedSchemaColumnType.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc
b/hudi-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc
rename to hudi-common/src/test/resources/exampleEvolvedSchemaDeleteColumn.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleSchema.avsc
b/hudi-common/src/test/resources/exampleSchema.avsc
similarity index 100%
rename from hudi-client/hudi-client-common/src/test/resources/exampleSchema.avsc
rename to hudi-common/src/test/resources/exampleSchema.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithMetaFields.avsc
b/hudi-common/src/test/resources/exampleSchemaWithMetaFields.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithMetaFields.avsc
rename to hudi-common/src/test/resources/exampleSchemaWithMetaFields.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithUDT.avsc
b/hudi-common/src/test/resources/exampleSchemaWithUDT.avsc
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/exampleSchemaWithUDT.avsc
rename to hudi-common/src/test/resources/exampleSchemaWithUDT.avsc
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile
b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile
rename to
hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_bootstrap_index_partitions.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile
b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile
rename to hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_complex.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile
b/hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile
rename to hudi-common/src/test/resources/hudi_0_10_hbase_1_2_3_simple.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile
b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile
rename to
hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_bootstrap_index_partitions.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile
b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile
rename to hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_complex.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile
b/hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile
rename to hudi-common/src/test/resources/hudi_0_11_hbase_2_4_9_simple.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile
b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile
rename to
hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_bootstrap_index_partitions.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile
b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile
rename to hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_complex.hfile
diff --git
a/hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile
b/hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile
similarity index 100%
rename from
hudi-client/hudi-client-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile
rename to hudi-common/src/test/resources/hudi_0_9_hbase_1_2_3_simple.hfile
