This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 92e52dca1ae [HUDI-6170] Use correct zone id while calculating
earliestTimeToRetain (#8631)
92e52dca1ae is described below
commit 92e52dca1ae08576e565a4ce8419b4d9c7354002
Author: Lokesh Jain <[email protected]>
AuthorDate: Wed May 10 08:07:39 2023 +0530
[HUDI-6170] Use correct zone id while calculating earliestTimeToRetain
(#8631)
* Use correct zone id while calculating earliestTimeToRetain
* Use metaClient table config
---
.../org/apache/hudi/client/HoodieTimelineArchiver.java | 3 +--
.../org/apache/hudi/table/action/clean/CleanPlanner.java | 3 +--
.../apache/hudi/common/model/HoodieTimelineTimeZone.java | 15 ++++++++++++---
.../org/apache/hudi/common/table/HoodieTableConfig.java | 4 ++++
.../hudi/common/table/timeline/TestHoodieInstant.java | 9 +++++++++
5 files changed, 27 insertions(+), 7 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
index 025aff09bdd..6463cc10375 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
@@ -76,7 +76,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.text.ParseException;
import java.time.Instant;
-import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
@@ -489,7 +488,7 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
String latestCommitToArchive =
instantsToArchive.get(instantsToArchive.size() - 1).getTimestamp();
try {
Instant latestCommitInstant =
HoodieActiveTimeline.parseDateFromInstantTime(commitTimeline.lastInstant().get().getTimestamp()).toInstant();
- ZonedDateTime currentDateTime =
ZonedDateTime.ofInstant(latestCommitInstant, ZoneId.systemDefault());
+ ZonedDateTime currentDateTime =
ZonedDateTime.ofInstant(latestCommitInstant,
metaClient.getTableConfig().getTimelineTimezone().getZoneId());
String earliestTimeToRetain =
HoodieActiveTimeline.formatDate(Date.from(currentDateTime.minusHours(config.getCleanerHoursRetained()).toInstant()));
if (HoodieTimeline.compareTimestamps(latestCommitToArchive,
GREATER_THAN_OR_EQUALS, earliestTimeToRetain)) {
throw new HoodieIOException("Please align your archival configs
based on cleaner configs. 'hoodie.keep.min.commits' : "
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index ae183b678e6..005e3f1dc34 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -53,7 +53,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
import java.time.Instant;
-import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
@@ -510,7 +509,7 @@ public class CleanPlanner<T, I, K, O> implements
Serializable {
}
} else if (config.getCleanerPolicy() ==
HoodieCleaningPolicy.KEEP_LATEST_BY_HOURS) {
Instant instant = Instant.now();
- ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(instant,
ZoneId.systemDefault());
+ ZonedDateTime currentDateTime = ZonedDateTime.ofInstant(instant,
hoodieTable.getMetaClient().getTableConfig().getTimelineTimezone().getZoneId());
String earliestTimeToRetain =
HoodieActiveTimeline.formatDate(Date.from(currentDateTime.minusHours(hoursRetained).toInstant()));
earliestCommitToRetain =
Option.fromJavaOptional(commitTimeline.getInstantsAsStream().filter(i ->
HoodieTimeline.compareTimestamps(i.getTimestamp(),
HoodieTimeline.GREATER_THAN_OR_EQUALS,
earliestTimeToRetain)).findFirst());
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
index 9b1c695d491..29e158812f4 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTimelineTimeZone.java
@@ -18,20 +18,29 @@
package org.apache.hudi.common.model;
+import java.time.ZoneId;
+import java.util.TimeZone;
+
/**
* Hoodie TimelineZone.
*/
public enum HoodieTimelineTimeZone {
- LOCAL("local"),
- UTC("utc");
+ LOCAL("local", ZoneId.systemDefault()),
+ UTC("utc", TimeZone.getTimeZone("UTC").toZoneId());
private final String timeZone;
+ private final ZoneId zoneId;
- HoodieTimelineTimeZone(String timeZone) {
+ HoodieTimelineTimeZone(String timeZone, ZoneId zoneId) {
this.timeZone = timeZone;
+ this.zoneId = zoneId;
}
public String getTimeZone() {
return timeZone;
}
+
+ public ZoneId getZoneId() {
+ return zoneId;
+ }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index a5e4456381b..c12a2da3e32 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -661,6 +661,10 @@ public class HoodieTableConfig extends HoodieConfig {
return getString(KEY_GENERATOR_CLASS_NAME);
}
+ public HoodieTimelineTimeZone getTimelineTimezone() {
+ return
HoodieTimelineTimeZone.valueOf(getStringOrDefault(TIMELINE_TIMEZONE));
+ }
+
public String getHiveStylePartitioningEnable() {
return getStringOrDefault(HIVE_STYLE_PARTITIONING_ENABLE);
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
index c4a1d00e90d..39d4040c93a 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieInstant.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.table.timeline;
+import org.apache.hudi.common.model.HoodieTimelineTimeZone;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.Option;
import org.junit.jupiter.api.Test;
@@ -27,6 +28,7 @@ import java.util.stream.Stream;
import static org.apache.hudi.common.testutils.Assertions.assertStreamEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
public class TestHoodieInstant extends HoodieCommonTestHarness {
@@ -76,4 +78,11 @@ public class TestHoodieInstant extends
HoodieCommonTestHarness {
cleanMetaClient();
}
}
+
+ @Test
+ public void testHoodieTimelineTimeZone() {
+ for (HoodieTimelineTimeZone timeZone : HoodieTimelineTimeZone.values()) {
+ assertNotNull(timeZone.getZoneId());
+ }
+ }
}
