[gobblin] branch master updated: [GOBBLIN-1494]Fix flaky test on HiveSource (#3338)

[aplex]

This is an automated email from the ASF dual-hosted git repository.

aplex pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git


The following commit(s) were added to refs/heads/master by this push:
     new 48af4c6  [GOBBLIN-1494]Fix flaky test on HiveSource (#3338)
48af4c6 is described below

commit 48af4c61c4780740b63931901c756e5a184e763b
Author: William Lo <lo.willia...@gmail.com>
AuthorDate: Thu Jul 22 17:57:22 2021 -0700

    [GOBBLIN-1494]Fix flaky test on HiveSource (#3338)
    
    HiveSourceTest returns 2 watermarks for testGetWorkunitsAfterWatermark 
instead of 1.
    This is due to System.getCurrentTime.millis() returning a different value 
when comparing against the filesystem last updated filetime. To guard against 
this, change the test to incorporate the workunit within the watermark by 
assigning it an earlier creation time, thus implying that the file was modified 
so it needs to be ingested again.
---
 .../data/management/conversion/hive/HiveSourceTest.java   | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git 
a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
 
b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
index 4a1da03..2351fd3 100644
--- 
a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
+++ 
b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/conversion/hive/HiveSourceTest.java
@@ -133,21 +133,30 @@ public class HiveSourceTest {
 
     Table table1 = 
this.hiveMetastoreTestUtils.getLocalMetastoreClient().getTable(dbName, 
TEST_TABLE_1);
 
+    // Denote watermark to have a past created timestamp, so that the 
watermark workunit gets generated
+    // This is so that the test is reliable across different operating systems 
and not flaky to System timing differences
     previousWorkUnitStates.add(ConversionHiveTestUtils.createWus(dbName, 
TEST_TABLE_1,
-        TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), 
TimeUnit.SECONDS)));
+        TimeUnit.MILLISECONDS.convert(table1.getCreateTime(), 
TimeUnit.SECONDS)-100));
 
     SourceState testState = new SourceState(getTestState(dbName), 
previousWorkUnitStates);
     testState.setProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, 
TableLevelWatermarker.Factory.class.getName());
 
     List<WorkUnit> workUnits = this.hiveSource.getWorkunits(testState);
 
-    Assert.assertEquals(workUnits.size(), 1);
+    Assert.assertEquals(workUnits.size(), 2);
     WorkUnit wu = workUnits.get(0);
 
     HiveWorkUnit hwu = new HiveWorkUnit(wu);
 
     Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getDb(), dbName);
-    Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), 
TEST_TABLE_2);
+    Assert.assertEquals(hwu.getHiveDataset().getDbAndTable().getTable(), 
TEST_TABLE_1);
+
+    WorkUnit wu2 = workUnits.get(1);
+
+    HiveWorkUnit hwu2 = new HiveWorkUnit(wu2);
+
+    Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getDb(), dbName);
+    Assert.assertEquals(hwu2.getHiveDataset().getDbAndTable().getTable(), 
TEST_TABLE_2);
   }
 
   @Test