Repository: incubator-gobblin Updated Branches: refs/heads/master c65010a8e -> 7d970606b
[GOBBLIN-275] Use listStatus instead of globStatus for finding persisted files Closes #2128 from jack-moseley/distcp_glob Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/7d970606 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/7d970606 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/7d970606 Branch: refs/heads/master Commit: 7d970606bd41eceff567d8bd557c3cdc521ea923 Parents: c65010a Author: Jack Moseley <[email protected]> Authored: Tue Oct 10 16:28:55 2017 -0700 Committer: Issac Buenrostro <[email protected]> Committed: Tue Oct 10 16:28:55 2017 -0700 ---------------------------------------------------------------------- .../copy/recovery/RecoveryHelper.java | 22 +++++++++++++------- .../copy/recovery/RecoveryHelperTest.java | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/7d970606/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelper.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelper.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelper.java index c193922..697637e 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelper.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelper.java @@ -19,6 +19,7 @@ package org.apache.gobblin.data.management.copy.recovery; import lombok.extern.slf4j.Slf4j; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.concurrent.TimeUnit; @@ -91,15 +92,13 @@ public class RecoveryHelper { } String guid = computeGuid(state, file); - StringBuilder nameBuilder = new StringBuilder(guid); - nameBuilder.append("_"); - nameBuilder.append(shortenPathName(file.getOrigin().getPath(), 250 - nameBuilder.length())); + Path guidPath = new Path(this.persistDir.get(), guid); - if (!this.fs.exists(this.persistDir.get())) { - this.fs.mkdirs(this.persistDir.get(), new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE)); + if (!this.fs.exists(guidPath)) { + this.fs.mkdirs(guidPath, new FsPermission(FsAction.ALL, FsAction.READ, FsAction.NONE)); } - Path targetPath = new Path(this.persistDir.get(), nameBuilder.toString()); + Path targetPath = new Path(guidPath, shortenPathName(file.getOrigin().getPath(), 250 - guid.length())); log.info(String.format("Persisting file %s with guid %s to location %s.", path, guid, targetPath)); if (this.fs.rename(path, targetPath)) { this.fs.setTimes(targetPath, System.currentTimeMillis(), -1); @@ -122,8 +121,15 @@ public class RecoveryHelper { return Optional.absent(); } - Path glob = new Path(this.persistDir.get(), computeGuid(state, file) + "_*"); - for (FileStatus fileStatus : this.fs.globStatus(glob)) { + Path guidPath = new Path(this.persistDir.get(), computeGuid(state, file)); + FileStatus[] statuses; + try { + statuses = this.fs.listStatus(guidPath); + } catch (FileNotFoundException e) { + return Optional.absent(); + } + + for (FileStatus fileStatus : statuses) { if (filter.apply(fileStatus)) { return Optional.of(fileStatus); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/7d970606/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelperTest.java ---------------------------------------------------------------------- diff --git a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelperTest.java b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelperTest.java index c825928..83627bb 100644 --- a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelperTest.java +++ b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/recovery/RecoveryHelperTest.java @@ -103,7 +103,7 @@ public class RecoveryHelperTest { Assert.assertEquals(stagingDir.listFiles().length, 0); Assert.assertEquals(recoveryDir.listFiles().length, 1); - File fileInRecovery = recoveryDir.listFiles()[0]; + File fileInRecovery = recoveryDir.listFiles()[0].listFiles()[0]; Assert.assertEquals(IOUtils.readLines(new FileInputStream(fileInRecovery)).get(0), content); Optional<FileStatus> fileToRecover =
