This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e712f4c8c3b Revert "[HUDI-6119] Add testcase for timeline timezone UTC
(#8528)" (#8549)
e712f4c8c3b is described below
commit e712f4c8c3bf27782dd39c9bc810da38b6dec185
Author: Danny Chan <[email protected]>
AuthorDate: Sun Apr 23 09:30:38 2023 +0800
Revert "[HUDI-6119] Add testcase for timeline timezone UTC (#8528)" (#8549)
This reverts commit 99eabd562c5ce2a8626708b67cf5617d3c84ce77.
---
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 48 +---------------------
1 file changed, 2 insertions(+), 46 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 14ffb125805..2c05a883fce 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -23,7 +23,6 @@ import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.HoodieSparkUtils.gteqSpark3_0
import org.apache.hudi.client.SparkRDDWriteClient
import org.apache.hudi.common.model._
-import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient,
TableSchemaResolver}
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.config.{HoodieBootstrapConfig, HoodieIndexConfig,
HoodieWriteConfig}
@@ -50,10 +49,8 @@ import org.scalatest.Assertions.assertThrows
import org.scalatest.Matchers.{be, convertToAnyShouldWrapper, intercept}
import java.io.IOException
-import java.time.format.DateTimeFormatterBuilder
-import java.time.{Instant, ZoneId}
-import java.time.temporal.ChronoField
-import java.util.{Collections, Date, TimeZone, UUID}
+import java.time.Instant
+import java.util.{Collections, Date, UUID}
import scala.collection.JavaConversions._
import scala.collection.JavaConverters
@@ -1211,47 +1208,6 @@ class TestHoodieSparkSqlWriter {
assert(exc.getMessage.contains("Consistent hashing bucket index does not
work with COW table. Use simple bucket index or an MOR table."))
}
- /*
- * Test case for instant is generated with commit timezone when
TIMELINE_TIMEZONE set to UTC
- * related to HUDI-5978
- */
- @Test
- def testInsertDatasetWIthTimelineTimezoneUTC(): Unit = {
- val fooTableModifier =
commonTableModifier.updated(DataSourceWriteOptions.OPERATION.key,
DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
- .updated(DataSourceWriteOptions.INSERT_DROP_DUPS.key, "false")
- .updated(HoodieTableConfig.TIMELINE_TIMEZONE.key, "UTC") // utc timezone
-
- // generate the inserts
- val schema = DataSourceTestUtils.getStructTypeExampleSchema
- val structType = AvroConversionUtils.convertAvroSchemaToStructType(schema)
- val records = DataSourceTestUtils.generateRandomRows(100)
- val recordsSeq = convertRowListToSeq(records)
- val df = spark.createDataFrame(sc.parallelize(recordsSeq), structType)
-
- // get UTC instant before write
- val beforeWriteInstant = Instant.now()
-
- // set local timezone to America/Los_Angeles(UTC-7)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
-
- // write to Hudi
- val (success, writeInstantTimeOpt, _, _, _, hoodieTableConfig) =
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, fooTableModifier, df)
- assertTrue(success)
- val hoodieTableTimelineTimezone =
HoodieTimelineTimeZone.valueOf(hoodieTableConfig.getString(HoodieTableConfig.TIMELINE_TIMEZONE))
- assertEquals(hoodieTableTimelineTimezone, HoodieTimelineTimeZone.UTC)
-
- val utcFormatter = new DateTimeFormatterBuilder()
- .appendPattern(HoodieInstantTimeGenerator.SECS_INSTANT_TIMESTAMP_FORMAT)
- .appendValue(ChronoField.MILLI_OF_SECOND, 3)
- .toFormatter
- .withZone(ZoneId.of("UTC"))
- // instant parsed by UTC timezone
- val writeInstant =
Instant.from(utcFormatter.parse(writeInstantTimeOpt.get()))
-
- assertTrue(beforeWriteInstant.toEpochMilli < writeInstant.toEpochMilli,
- s"writeInstant(${writeInstant.toEpochMilli}) must always be greater than
beforeWriteInstant(${beforeWriteInstant.toEpochMilli}) if writeInstant was
generated with UTC timezone")
- }
-
private def fetchActualSchema(): Schema = {
val tableMetaClient = HoodieTableMetaClient.builder()
.setConf(spark.sparkContext.hadoopConfiguration)
