[GOBBLIN-410] Support REPLACE_TABLE_AND_PARTITIONS for Hive copies. Closes #2287 from ibuenros/replace-table-and- partitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/11182dcf Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/11182dcf Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/11182dcf Branch: refs/heads/0.12.0 Commit: 11182dcf3ed771a38fbbaa350cb115cf5e0fe3e3 Parents: 4f52230 Author: ibuenros <[email protected]> Authored: Tue Feb 20 10:35:59 2018 -0800 Committer: Hung Tran <[email protected]> Committed: Tue Feb 20 10:35:59 2018 -0800 ---------------------------------------------------------------------- .../gobblin/data/management/copy/hive/HiveCopyEntityHelper.java | 2 ++ .../gobblin/data/management/copy/hive/HivePartitionFileSet.java | 3 ++- .../data/management/copy/hive/UnpartitionedTableFileSet.java | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java index 2580775..2b1a142 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java @@ -200,6 +200,8 @@ public class HiveCopyEntityHelper { REPLACE_PARTITIONS, /** Deregister target table, do NOT delete its files, and create a new table with correct values. */ REPLACE_TABLE, + /** A combination of {@link #REPLACE_TABLE} and {@link #REPLACE_PARTITIONS}*/ + REPLACE_TABLE_AND_PARTITIONS, /** Keep the target table as registered while updating the file location */ UPDATE_TABLE, /** Abort copying of conflict table. */ http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java index 34b6933..2c3817e 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/HivePartitionFileSet.java @@ -96,7 +96,8 @@ public class HivePartitionFileSet extends HiveFileSet { try { checkPartitionCompatibility(targetPartition, this.existingTargetPartition.get()); } catch (IOException ioe) { - if (hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_PARTITIONS) { + if (hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_PARTITIONS && + hiveCopyEntityHelper.getExistingEntityPolicy() != HiveCopyEntityHelper.ExistingEntityPolicy.REPLACE_TABLE_AND_PARTITIONS) { log.error("Source and target partitions are not compatible. Aborting copy of partition " + this.partition, ioe); return Lists.newArrayList(); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/11182dcf/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java index 21813fb..756b4dd 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java @@ -73,6 +73,7 @@ public class UnpartitionedTableFileSet extends HiveFileSet { existingTargetTable = Optional.absent(); break ; case REPLACE_TABLE: + case REPLACE_TABLE_AND_PARTITIONS: // Required to de-register the original table. log.warn("Source and target table are not compatible. Will override target table " + existingTargetTable.get() .getDataLocation());
