This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9ef51de Add empty payload class to support deletes via apache spark
9ef51de is described below
commit 9ef51deb841765e85a1d1c8ab337677a687dfb66
Author: lyogev <[email protected]>
AuthorDate: Thu Apr 11 15:14:53 2019 +0300
Add empty payload class to support deletes via apache spark
---
.../com/uber/hoodie/EmptyHoodieRecordPayload.java | 48 ++++++++++++++++++++++
.../src/test/scala/DataSourceDefaultsTest.scala | 16 +++++++-
2 files changed, 63 insertions(+), 1 deletion(-)
diff --git
a/hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java
b/hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java
new file mode 100644
index 0000000..85eb5ff
--- /dev/null
+++ b/hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019 Uber Technologies, Inc. ([email protected])
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ */
+
+package com.uber.hoodie;
+
+import com.uber.hoodie.common.model.HoodieRecordPayload;
+import java.util.Optional;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+
+/**
+ * Empty payload used for deletions
+ */
+public class EmptyHoodieRecordPayload implements
HoodieRecordPayload<EmptyHoodieRecordPayload> {
+
+ public EmptyHoodieRecordPayload(GenericRecord record, Comparable
orderingVal) { }
+
+ @Override
+ public EmptyHoodieRecordPayload preCombine(EmptyHoodieRecordPayload another)
{
+ return another;
+ }
+
+ @Override
+ public Optional<IndexedRecord> combineAndGetUpdateValue(IndexedRecord
currentValue, Schema schema) {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<IndexedRecord> getInsertValue(Schema schema) {
+ return Optional.empty();
+ }
+}
diff --git a/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala
b/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala
index 2abf50c..14960b1 100644
--- a/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala
+++ b/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala
@@ -16,9 +16,11 @@
*
*/
+import java.util.Optional
+
import com.uber.hoodie.common.util.{SchemaTestUtil, TypedProperties}
import com.uber.hoodie.exception.HoodieException
-import com.uber.hoodie.{DataSourceWriteOptions,
OverwriteWithLatestAvroPayload, SimpleKeyGenerator}
+import com.uber.hoodie.{DataSourceWriteOptions, EmptyHoodieRecordPayload,
OverwriteWithLatestAvroPayload, SimpleKeyGenerator}
import org.apache.avro.generic.GenericRecord
import org.junit.Assert._
import org.junit.{Before, Test}
@@ -114,4 +116,16 @@ class DataSourceDefaultsTest extends AssertionsForJUnit {
val combinedGR21 =
combinedPayload21.getInsertValue(schema).get().asInstanceOf[GenericRecord]
assertEquals("field2", combinedGR21.get("field1").toString)
}
+
+ @Test def testEmptyHoodieRecordPayload() = {
+ val emptyPayload1 = new EmptyHoodieRecordPayload(baseRecord, 1)
+ val laterRecord = SchemaTestUtil
+ .generateAvroRecordFromJson(schema, 2, "001", "f1")
+ val emptyPayload2 = new EmptyHoodieRecordPayload(laterRecord, 2)
+
+ // it will provide an empty record
+ val combinedPayload12 = emptyPayload1.preCombine(emptyPayload2)
+ val combined12 = combinedPayload12.getInsertValue(schema)
+ assertEquals(Optional.empty(), combined12)
+ }
}
