Repository: incubator-gobblin Updated Branches: refs/heads/master 169a6a5e7 -> d1950135e
[GOBBLIN-267] Changed workunit creation policy to compare update time with maxLookBackDays Closes #2119 from aditya1105/avro-to-orc Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/d1950135 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/d1950135 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/d1950135 Branch: refs/heads/master Commit: d1950135ed8f1db17020f4f8f634ff01c3dc2629 Parents: 169a6a5 Author: aditya1105 <[email protected]> Authored: Tue Sep 26 21:22:08 2017 -0700 Committer: Hung Tran <[email protected]> Committed: Tue Sep 26 21:22:08 2017 -0700 ---------------------------------------------------------------------- .../data/management/conversion/hive/source/HiveSource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/d1950135/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java index b71d6b6..c72ee91 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java @@ -386,10 +386,13 @@ public class HiveSource implements Source { /** * Check if workunit needs to be created. Returns <code>true</code> If the - * <code>updateTime</code> is greater than the <code>lowWatermark</code>. + * <code>updateTime</code> is greater than the <code>lowWatermark</code> and <code>maxLookBackTime</code> * <code>createTime</code> is not used. It exists for backward compatibility */ protected boolean shouldCreateWorkunit(long createTime, long updateTime, LongWatermark lowWatermark) { + if (new DateTime(updateTime).isBefore(this.maxLookBackTime)) { + return false; + } return new DateTime(updateTime).isAfter(lowWatermark.getValue()); }
