[GOBBLIN-399] Refactor HiveSource#shouldCreateWorkunit() to accept table as parameter
Closes #2273 from aditya1105/metadata Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/af68d7ed Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/af68d7ed Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/af68d7ed Branch: refs/heads/0.12.0 Commit: af68d7edd788685279e6e0cf2ad0b6cbf8e7f2d0 Parents: d29b72f Author: aditya1105 <[email protected]> Authored: Thu Feb 1 11:38:08 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Thu Feb 1 11:38:08 2018 -0800 ---------------------------------------------------------------------- .../data/management/conversion/hive/source/HiveSource.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/af68d7ed/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java index 94d5ad9..3ad99fd 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java @@ -238,7 +238,7 @@ public class HiveSource implements Source { return; } - if (shouldCreateWorkunit(getCreateTime(hiveDataset.getTable()), updateTime, lowWatermark)) { + if (shouldCreateWorkunit(hiveDataset.getTable(), lowWatermark)) { log.info(String.format( "Creating workunit for table %s as updateTime %s or createTime %s is greater than low watermark %s", @@ -384,6 +384,13 @@ public class HiveSource implements Source { return shouldCreateWorkunit(createTime, updateTime, lowWatermark); } + protected boolean shouldCreateWorkunit(Table table, LongWatermark lowWatermark) + throws UpdateNotFoundException { + long updateTime = this.updateProvider.getUpdateTime(table); + long createTime = getCreateTime(table); + return shouldCreateWorkunit(createTime, updateTime, lowWatermark); + } + /** * Check if workunit needs to be created. Returns <code>true</code> If the * <code>updateTime</code> is greater than the <code>lowWatermark</code> and <code>maxLookBackTime</code>
