This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new aa0da72 Preparation for Avro update (#2650)
aa0da72 is described below
commit aa0da72c59cb3764205f90b025b24d1640727795
Author: Sebastian Bernauer <[email protected]>
AuthorDate: Wed Mar 31 06:50:17 2021 +0200
Preparation for Avro update (#2650)
---
.../java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java | 6 ++----
.../src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java | 4 ++--
hudi-common/src/main/avro/HoodieClusteringPlan.avsc | 2 +-
hudi-common/src/main/avro/HoodieFSPermission.avsc | 8 ++++----
hudi-common/src/main/avro/HoodieRequestedReplaceMetadata.avsc | 4 ++--
hudi-common/src/main/avro/HoodieRestoreMetadata.avsc | 2 +-
.../java/org/apache/hudi/common/table/TableSchemaResolver.java | 2 +-
.../src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java | 3 +--
8 files changed, 14 insertions(+), 17 deletions(-)
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java
index 4df0ac4..a8f0aac 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java
@@ -34,8 +34,6 @@ import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.HoodieKeyLocationFetchHandle;
import org.apache.hudi.table.HoodieTable;
-import avro.shaded.com.google.common.collect.Lists;
-
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -135,8 +133,8 @@ public class FlinkHoodieSimpleIndex<T extends
HoodieRecordPayload> extends Flink
context.map(latestBaseFiles, partitionPathBaseFile -> new
HoodieKeyLocationFetchHandle<>(config, hoodieTable, partitionPathBaseFile),
parallelism);
Map<HoodieKey, HoodieRecordLocation> recordLocations = new HashMap<>();
hoodieKeyLocationFetchHandles.stream()
- .flatMap(handle -> Lists.newArrayList(handle.locations()).stream())
- .forEach(x -> x.forEach(y -> recordLocations.put(y.getKey(),
y.getRight())));
+ .flatMap(handle -> handle.locations())
+ .forEach(x -> recordLocations.put(x.getKey(), x.getRight()));
return recordLocations;
}
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java
index 31ad241..e42b6ba 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java
@@ -18,7 +18,6 @@
package org.apache.hudi.index.hbase;
-import avro.shaded.com.google.common.collect.Maps;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
@@ -65,6 +64,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.HashMap;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -469,7 +469,7 @@ public class TestHBaseIndex extends FunctionalTestHarness {
@Test
public void testHbaseTagLocationForArchivedCommits() throws Exception {
// Load to memory
- Map<String, String> params = Maps.newHashMap();
+ Map<String, String> params = new HashMap<String, String>();
params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, "1");
params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP, "3");
params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP, "2");
diff --git a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
index 709a0eb..bc8ed0e 100644
--- a/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
+++ b/hudi-common/src/main/avro/HoodieClusteringPlan.avsc
@@ -30,7 +30,7 @@
},
{
"name":"strategy",
- "type":["HoodieClusteringStrategy", "null"],
+ "type":["null", "HoodieClusteringStrategy"],
"default": null
},
{
diff --git a/hudi-common/src/main/avro/HoodieFSPermission.avsc
b/hudi-common/src/main/avro/HoodieFSPermission.avsc
index e589335..b64b16c 100644
--- a/hudi-common/src/main/avro/HoodieFSPermission.avsc
+++ b/hudi-common/src/main/avro/HoodieFSPermission.avsc
@@ -28,22 +28,22 @@
{
"name":"userAction",
"type":[ "null", "string" ],
- "default": "null"
+ "default": null
},
{
"name":"groupAction",
"type":[ "null", "string" ],
- "default": "null"
+ "default": null
},
{
"name":"otherAction",
"type":[ "null", "string" ],
- "default": "null"
+ "default": null
},
{
"name":"stickyBit",
"type":[ "null", "boolean" ],
- "default": "null"
+ "default": null
}
]
}
diff --git a/hudi-common/src/main/avro/HoodieRequestedReplaceMetadata.avsc
b/hudi-common/src/main/avro/HoodieRequestedReplaceMetadata.avsc
index f98f424..bc06780 100644
--- a/hudi-common/src/main/avro/HoodieRequestedReplaceMetadata.avsc
+++ b/hudi-common/src/main/avro/HoodieRequestedReplaceMetadata.avsc
@@ -23,11 +23,11 @@
{
"name":"operationType",
"type":["null", "string"],
- "default": ""
+ "default": null
},
{
"name":"clusteringPlan", /* only set if operationType == clustering" */
- "type":["HoodieClusteringPlan", "null"],
+ "type":["null", "HoodieClusteringPlan"],
"default": null
},
{
diff --git a/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc
b/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc
index 6c6f98f..f9795d3 100644
--- a/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc
+++ b/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc
@@ -38,7 +38,7 @@
/* overlaps with 'instantsToRollback' field. Adding this to track action
type for all the instants being rolled back. */
{
"name": "restoreInstantInfo",
- "default": null,
+ "default": [],
"type": {
"type": "array",
"default": null,
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index 932a455..9467bfd 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -330,7 +330,7 @@ public class TableSchemaResolver {
for (final Field newSchemaField : newSchema.getFields()) {
final Field oldSchemaField =
SchemaCompatibility.lookupWriterField(oldSchema, newSchemaField);
if (oldSchemaField == null) {
- if (newSchemaField.defaultValue() == null) {
+ if (newSchemaField.defaultVal() == null) {
// C3: newly added field in newSchema does not have a default value
return false;
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
index 863103e..8887cfe 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
@@ -25,7 +25,6 @@ import org.apache.hudi.exception.SchemaCompatibilityException;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
-import org.codehaus.jackson.node.NullNode;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
@@ -184,7 +183,7 @@ public class TestHoodieAvroUtils {
Schema.Field evolvedField1 = new Schema.Field("key",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
Schema.Field evolvedField2 = new Schema.Field("key1",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
Schema.Field evolvedField3 = new Schema.Field("key2",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
- Schema.Field evolvedField4 = new Schema.Field("evolved_field",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", NullNode.getInstance());
+ Schema.Field evolvedField4 = new Schema.Field("evolved_field",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
Schema.Field evolvedField5 = new Schema.Field("evolved_field1",
HoodieAvroUtils.METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
evolvedFields.add(evolvedField1);
evolvedFields.add(evolvedField2);
