This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1c8611a48c4 [HUDI-6440] Fallback to default Marker Type if the content
of marker file is empty (#9061)
1c8611a48c4 is described below
commit 1c8611a48c459ab2d1b88bcb9c810334192fa117
Author: zain <[email protected]>
AuthorDate: Wed Jun 28 14:44:32 2023 +0800
[HUDI-6440] Fallback to default Marker Type if the content of marker file
is empty (#9061)
When the content of marker file is empty, fallback to default marker type
instead.
---
.../org/apache/hudi/common/util/MarkerUtils.java | 6 +-
.../apache/hudi/common/util/TestMarkerUtils.java | 89 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 1 deletion(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/MarkerUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/MarkerUtils.java
index ceff967ef9a..73ad7e7dfc7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/MarkerUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/MarkerUtils.java
@@ -118,7 +118,11 @@ public class MarkerUtils {
return Option.empty();
}
fsDataInputStream = fileSystem.open(markerTypeFilePath);
- content =
Option.of(MarkerType.valueOf(FileIOUtils.readAsUTFString(fsDataInputStream)));
+ String markerType = FileIOUtils.readAsUTFString(fsDataInputStream);
+ if (StringUtils.isNullOrEmpty(markerType)) {
+ return Option.empty();
+ }
+ content = Option.of(MarkerType.valueOf(markerType));
} catch (IOException e) {
throw new HoodieIOException("Cannot read marker type file " +
markerTypeFilePath.toString()
+ "; " + e.getMessage(), e);
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/util/TestMarkerUtils.java
b/hudi-common/src/test/java/org/apache/hudi/common/util/TestMarkerUtils.java
new file mode 100644
index 00000000000..68660b117ce
--- /dev/null
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestMarkerUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.table.marker.MarkerType;
+import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
+import org.apache.hudi.exception.HoodieException;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.apache.hudi.common.util.MarkerUtils.MARKER_TYPE_FILENAME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class TestMarkerUtils extends HoodieCommonTestHarness {
+
+ private FileSystem fs;
+
+ @BeforeEach
+ public void setup() {
+ initPath();
+ fs = FSUtils.getFs(basePath, new Configuration());
+ }
+
+ @Test
+ public void testReadMarkerType() throws IOException {
+ // mock markers file
+ String markerDir = this.basePath + "/.hoodie/.temp/testReadMarkerType/";
+ if (MarkerUtils.doesMarkerTypeFileExist(fs, markerDir)) {
+ MarkerUtils.deleteMarkerTypeFile(fs, markerDir);
+ }
+
+ try {
+ // marker file does not exist
+ assertEquals(Option.empty(), MarkerUtils.readMarkerType(fs, markerDir),
+ "File does not exist, should be empty");
+
+ // HUDI-6440: Fallback to default Marker Type if the content of marker
file is empty
+ assertTrue(writeEmptyMarkerTypeToFile(fs, markerDir), "Failed to create
empty marker type file");
+ assertEquals(Option.empty(), MarkerUtils.readMarkerType(fs, markerDir),
+ "File exists but empty, should be empty");
+
+ // marker type is DIRECT
+ MarkerUtils.deleteMarkerTypeFile(fs, markerDir);
+ MarkerUtils.writeMarkerTypeToFile(MarkerType.DIRECT, fs, markerDir);
+ assertEquals(Option.of(MarkerType.DIRECT),
MarkerUtils.readMarkerType(fs, markerDir),
+ "File exists and contains DIRECT, should be DIRECT");
+
+ // marker type is TIMELINE_SERVER_BASED
+ MarkerUtils.deleteMarkerTypeFile(fs, markerDir);
+ MarkerUtils.writeMarkerTypeToFile(MarkerType.TIMELINE_SERVER_BASED, fs,
markerDir);
+ assertEquals(Option.of(MarkerType.TIMELINE_SERVER_BASED),
MarkerUtils.readMarkerType(fs, markerDir),
+ "File exists and contains TIMELINE_SERVER_BASED, should be
TIMELINE_SERVER_BASED");
+ } finally {
+ MarkerUtils.deleteMarkerTypeFile(fs, markerDir);
+ }
+ }
+
+ private boolean writeEmptyMarkerTypeToFile(FileSystem fileSystem, String
markerDir) {
+ Path markerTypeFilePath = new Path(markerDir, MARKER_TYPE_FILENAME);
+ try {
+ return fileSystem.createNewFile(markerTypeFilePath);
+ } catch (IOException e) {
+ throw new HoodieException("Failed to create marker type file " +
markerTypeFilePath, e);
+ }
+ }
+}
