Repository: incubator-gobblin Updated Branches: refs/heads/master 056888e02 -> f3eadceed
[GOBBLIN-297] Changed access to protected for HiveSource and Watermarkers Closes #2151 from aditya1105/HiveSource Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/f3eadcee Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/f3eadcee Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/f3eadcee Branch: refs/heads/master Commit: f3eadceed0173f9b6b122f6e714531700d22941b Parents: 056888e Author: adsharma <[email protected]> Authored: Wed Oct 25 14:43:29 2017 +0530 Committer: Abhishek Tiwari <[email protected]> Committed: Wed Oct 25 14:43:29 2017 +0530 ---------------------------------------------------------------------- .../conversion/hive/source/HiveSource.java | 32 ++++++++++---------- .../watermarker/PartitionLevelWatermarker.java | 14 ++++----- .../hive/watermarker/TableLevelWatermarker.java | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java index c72ee91..94d5ad9 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/source/HiveSource.java @@ -137,19 +137,19 @@ public class HiveSource implements Source { public static final Gson GENERICS_AWARE_GSON = GsonInterfaceAdapter.getGson(Object.class); public static final Splitter COMMA_BASED_SPLITTER = Splitter.on(",").omitEmptyStrings().trimResults(); - private MetricContext metricContext; - private EventSubmitter eventSubmitter; - private AvroSchemaManager avroSchemaManager; - - private HiveUnitUpdateProvider updateProvider; - private HiveSourceWatermarker watermarker; - private IterableDatasetFinder<HiveDataset> datasetFinder; - private List<WorkUnit> workunits; - private long maxLookBackTime; - private long beginGetWorkunitsTime; - private List<String> ignoreDataPathIdentifierList; - - private final ClassAliasResolver<HiveBaseExtractorFactory> classAliasResolver = + protected MetricContext metricContext; + protected EventSubmitter eventSubmitter; + protected AvroSchemaManager avroSchemaManager; + + protected HiveUnitUpdateProvider updateProvider; + protected HiveSourceWatermarker watermarker; + protected IterableDatasetFinder<HiveDataset> datasetFinder; + protected List<WorkUnit> workunits; + protected long maxLookBackTime; + protected long beginGetWorkunitsTime; + protected List<String> ignoreDataPathIdentifierList; + + protected final ClassAliasResolver<HiveBaseExtractorFactory> classAliasResolver = new ClassAliasResolver<>(HiveBaseExtractorFactory.class); @Override @@ -219,7 +219,7 @@ public class HiveSource implements Source { } - private void createWorkunitForNonPartitionedTable(HiveDataset hiveDataset) throws IOException { + protected void createWorkunitForNonPartitionedTable(HiveDataset hiveDataset) throws IOException { // Create workunits for tables try { @@ -280,7 +280,7 @@ public class HiveSource implements Source { return hiveWorkUnit; } - private void createWorkunitsForPartitionedTable(HiveDataset hiveDataset, AutoReturnableObject<IMetaStoreClient> client) throws IOException { + protected void createWorkunitsForPartitionedTable(HiveDataset hiveDataset, AutoReturnableObject<IMetaStoreClient> client) throws IOException { long tableProcessTime = new DateTime().getMillis(); this.watermarker.onTableProcessBegin(hiveDataset.getTable(), tableProcessTime); @@ -425,7 +425,7 @@ public class HiveSource implements Source { } // Convert createTime from seconds to milliseconds - private static long getCreateTime(Table table) { + protected static long getCreateTime(Table table) { return TimeUnit.MILLISECONDS.convert(table.getTTable().getCreateTime(), TimeUnit.SECONDS); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java index 4aacb26..5fd3f76 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/PartitionLevelWatermarker.java @@ -112,31 +112,31 @@ public class PartitionLevelWatermarker implements HiveSourceWatermarker { @Setter(AccessLevel.PACKAGE) @VisibleForTesting - private long leastWatermarkToPersistInState; + protected long leastWatermarkToPersistInState; // Keep an additional 2 days of updates - private static final int BUFFER_WATERMARK_DAYS_TO_PERSIST = 2; + protected static final int BUFFER_WATERMARK_DAYS_TO_PERSIST = 2; /** * Watermarks from previous state */ @Getter(AccessLevel.PACKAGE) @VisibleForTesting - private final TableWatermarks previousWatermarks; + protected final TableWatermarks previousWatermarks; /** * Current expected watermarks */ @Getter(AccessLevel.PACKAGE) @VisibleForTesting - private final TableWatermarks expectedHighWatermarks; + protected final TableWatermarks expectedHighWatermarks; - private final HiveMetastoreClientPool pool; + protected final HiveMetastoreClientPool pool; /** * Delegates watermarking logic to {@link TableLevelWatermarker} for Non partitioned tables */ - private final TableLevelWatermarker tableLevelWatermarker; + protected final TableLevelWatermarker tableLevelWatermarker; - private final HiveUnitUpdateProvider updateProvider; + protected final HiveUnitUpdateProvider updateProvider; /** * Reads and initialized the previous high watermarks from {@link SourceState#getPreviousDatasetStatesByUrns()} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/f3eadcee/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java index c325d69..716d296 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/conversion/hive/watermarker/TableLevelWatermarker.java @@ -53,7 +53,7 @@ public class TableLevelWatermarker implements HiveSourceWatermarker { public static final Gson GSON = new Gson(); // Table complete name db@tb - list of previous workunitState - private Map<String, LongWatermark> tableWatermarks; + protected Map<String, LongWatermark> tableWatermarks; public TableLevelWatermarker(State state) { this.tableWatermarks = Maps.newHashMap();
