Repository: incubator-gobblin Updated Branches: refs/heads/master 467fe8fc8 -> cec91e610
[GOBBLIN-174] fix distcp-ng, so it does not remove existing target files fixing fullPathdiff() so it does not remove existing target files incase NO_DELETE delete method is specified clearing existingTargetTable if 'existing entity policy' is set to 'update_table' Closes #2008 from arjun4084346/distcpNgFixNoDelete Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/cec91e61 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/cec91e61 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/cec91e61 Branch: refs/heads/master Commit: cec91e61015568c6afb86c60f27181b7cc346a77 Parents: 467fe8f Author: Arjun <[email protected]> Authored: Fri Jul 28 19:31:22 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Fri Jul 28 19:31:22 2017 -0700 ---------------------------------------------------------------------- .../gobblin/data/management/copy/hive/HiveCopyEntityHelper.java | 2 +- .../data/management/copy/hive/UnpartitionedTableFileSet.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/cec91e61/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java index 230a02a..7cbd7bd 100644 --- a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java +++ b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/HiveCopyEntityHelper.java @@ -619,7 +619,7 @@ public class HiveCopyEntityHelper { if (shouldCopy) { builder.copyFile(sourcePath); } else { - // if not copying, we want to keep the file in the target + // If not copying, we want to keep the file in the target // at the end of this loop, all files in targetExistingPaths will be marked for deletion, so remove this file targetExistingPaths.remove(newPath); desiredTargetExistingPaths.remove(newPath); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/cec91e61/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java index 9d7291a..1481535 100644 --- a/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java +++ b/gobblin-data-management/src/main/java/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java @@ -70,6 +70,7 @@ public class UnpartitionedTableFileSet extends HiveFileSet { // Update the location of files while keep the existing table entity. log.warn("Source table will not be deregistered while file locaiton has been changed, update source table's" + " file location to" + this.helper.getTargetTable().getDataLocation()); + existingTargetTable = Optional.absent(); break ; case REPLACE_TABLE: // Required to de-register the original table.
