This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9796de9d4c [HUDI-4464] Clear warnings in Azure CI (#6210)
9796de9d4c is described below
commit 9796de9d4c9e173689a573c4ae384dee7792c982
Author: 冯健 <[email protected]>
AuthorDate: Sun Aug 7 13:30:03 2022 +0800
[HUDI-4464] Clear warnings in Azure CI (#6210)
Co-authored-by: jian.feng <[email protected]>
---
azure-pipelines.yml | 26 ++++++++--------
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 35 ++++++++++++++--------
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 8 +++++
3 files changed, 43 insertions(+), 26 deletions(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 1cc90da3ae..7cf83fed50 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -91,7 +91,7 @@ stages:
displayName: UT FT common & flink & UT client/spark-client
timeoutInMinutes: '150'
steps:
- - task: Maven@3
+ - task: [email protected]
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
@@ -99,7 +99,7 @@ stages:
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
- - task: Maven@3
+ - task: [email protected]
displayName: UT common flink client/spark-client
inputs:
mavenPomFile: 'pom.xml'
@@ -108,7 +108,7 @@ stages:
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g'
- - task: Maven@3
+ - task: [email protected]
displayName: FT common flink
inputs:
mavenPomFile: 'pom.xml'
@@ -121,7 +121,7 @@ stages:
displayName: FT client/spark-client
timeoutInMinutes: '150'
steps:
- - task: Maven@3
+ - task: [email protected]
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
@@ -129,7 +129,7 @@ stages:
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
- - task: Maven@3
+ - task: [email protected]
displayName: FT client/spark-client
inputs:
mavenPomFile: 'pom.xml'
@@ -142,7 +142,7 @@ stages:
displayName: UT FT clients & cli & utilities & sync
timeoutInMinutes: '150'
steps:
- - task: Maven@3
+ - task: [email protected]
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
@@ -150,7 +150,7 @@ stages:
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
- - task: Maven@3
+ - task: [email protected]
displayName: UT clients & cli & utilities & sync
inputs:
mavenPomFile: 'pom.xml'
@@ -159,7 +159,7 @@ stages:
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g'
- - task: Maven@3
+ - task: [email protected]
displayName: FT clients & cli & utilities & sync
inputs:
mavenPomFile: 'pom.xml'
@@ -172,7 +172,7 @@ stages:
displayName: UT FT other modules
timeoutInMinutes: '150'
steps:
- - task: Maven@3
+ - task: [email protected]
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
@@ -180,7 +180,7 @@ stages:
options: $(MVN_OPTS_INSTALL)
publishJUnitResults: false
jdkVersionOption: '1.8'
- - task: Maven@3
+ - task: [email protected]
displayName: UT other modules
inputs:
mavenPomFile: 'pom.xml'
@@ -189,7 +189,7 @@ stages:
publishJUnitResults: false
jdkVersionOption: '1.8'
mavenOptions: '-Xmx4g'
- - task: Maven@3
+ - task: [email protected]
displayName: FT other modules
inputs:
mavenPomFile: 'pom.xml'
@@ -202,7 +202,7 @@ stages:
displayName: IT modules
timeoutInMinutes: '150'
steps:
- - task: Maven@3
+ - task: [email protected]
displayName: maven install
inputs:
mavenPomFile: 'pom.xml'
@@ -210,7 +210,7 @@ stages:
options: $(MVN_OPTS_INSTALL) -Pintegration-tests
publishJUnitResults: false
jdkVersionOption: '1.8'
- - task: Maven@3
+ - task: [email protected]
displayName: UT integ-test
inputs:
mavenPomFile: 'pom.xml'
diff --git
a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
index 1daf638f2b..af6478e56e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -64,6 +64,8 @@ import java.nio.charset.StandardCharsets;
import java.sql.Date;
import java.sql.Timestamp;
import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -957,21 +959,28 @@ public class HoodieAvroUtils {
throw new AvroRuntimeException(String.format("cannot support rewrite value
for schema type: %s since the old schema type is: %s", newSchema, oldSchema));
}
- // convert days to Date
- private static java.sql.Date toJavaDate(int days) {
- long localMillis = Math.multiplyExact(days, MILLIS_PER_DAY);
- int timeZoneOffset;
- TimeZone defaultTimeZone = TimeZone.getDefault();
- if (defaultTimeZone instanceof sun.util.calendar.ZoneInfo) {
- timeZoneOffset = ((sun.util.calendar.ZoneInfo)
defaultTimeZone).getOffsetsByWall(localMillis, null);
- } else {
- timeZoneOffset = defaultTimeZone.getOffset(localMillis -
defaultTimeZone.getRawOffset());
- }
- return new java.sql.Date(localMillis - timeZoneOffset);
+ /**
+ * convert days to Date
+ *
+ * NOTE: This method could only be used in tests
+ *
+ * @VisibleForTesting
+ */
+ public static java.sql.Date toJavaDate(int days) {
+ LocalDate date = LocalDate.ofEpochDay(days);
+ ZoneId defaultZoneId = ZoneId.systemDefault();
+ ZonedDateTime zonedDateTime = date.atStartOfDay(defaultZoneId);
+ return new java.sql.Date(zonedDateTime.toInstant().toEpochMilli());
}
- // convert Date to days
- private static int fromJavaDate(Date date) {
+ /**
+ * convert Date to days
+ *
+ * NOTE: This method could only be used in tests
+ *
+ * @VisibleForTesting
+ */
+ public static int fromJavaDate(Date date) {
long millisUtc = date.getTime();
long millisLocal = millisUtc + TimeZone.getDefault().getOffset(millisUtc);
int julianDays = Math.toIntExact(Math.floorDiv(millisLocal,
MILLIS_PER_DAY));
diff --git
a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
index f2c02d627f..ad77e13b46 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
@@ -32,6 +32,7 @@ import org.junit.jupiter.api.Assertions;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.sql.Date;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -372,4 +373,11 @@ public class TestHoodieAvroUtils {
GenericRecord studentRecordRename =
HoodieAvroUtils.rewriteRecordWithNewSchema(rec3, nestedSchemaRename,
colRenames);
Assertions.assertEquals(GenericData.get().validate(nestedSchemaRename,
studentRecordRename), true);
}
+
+ @Test
+ public void testConvertDaysToDate() {
+ Date now = new Date(System.currentTimeMillis());
+ int days = HoodieAvroUtils.fromJavaDate(now);
+ assertEquals(now.toLocalDate(),
HoodieAvroUtils.toJavaDate(days).toLocalDate());
+ }
}
