Repository: falcon Updated Branches: refs/heads/0.6.1 7ec516ed4 -> 8cbe93e9c
FALCON-1146 feed retention policy deleted everything all the way up to the root. Contributed by Peeyush Bishnoi Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8cbe93e9 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8cbe93e9 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8cbe93e9 Branch: refs/heads/0.6.1 Commit: 8cbe93e9cf6bbd3ed46ba5bec4d2bc45c696f377 Parents: 7ec516e Author: Suhas Vasu <[email protected]> Authored: Mon Apr 13 23:42:02 2015 +0530 Committer: Suhas Vasu <[email protected]> Committed: Mon Apr 13 23:42:02 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../apache/falcon/entity/FileSystemStorage.java | 2 +- .../falcon/retention/FeedEvictorTest.java | 61 +++++++++++++++----- 3 files changed, 52 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 77a0fef..02b8be2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -137,6 +137,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1) (Suhas vasu) BUG FIXES + FALCON-1146 feed retention policy deleted everything all the way + up to the root (Peeyush Bishnoi via Suhas Vasu) + FALCON-1129 In a secure cluster, feed replication fails because of Authentication issues (Venkat Ranganathan via Suhas Vasu) http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java index 8948ad4..9ff33e7 100644 --- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java +++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java @@ -328,7 +328,7 @@ public class FileSystemStorage extends Configured implements Storage { DateFormat dateFormat = new SimpleDateFormat(FeedHelper.FORMAT); dateFormat.setTimeZone(timeZone); - Path feedBasePath = FeedHelper.getFeedBasePath(feedPath); + Path feedBasePath = fs.makeQualified(FeedHelper.getFeedBasePath(feedPath)); for (Path path : toBeDeleted) { deleteInstance(fs, path, feedBasePath); Date date = FeedHelper.getDate(feedPath, new Path(path.toUri().getPath()), timeZone); http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java ---------------------------------------------------------------------- diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java index a2feccf..e3daf8c 100644 --- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java +++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java @@ -24,6 +24,7 @@ import org.apache.falcon.entity.Storage; import org.apache.falcon.entity.v0.feed.LocationType; import org.apache.falcon.hadoop.HadoopClientFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -32,19 +33,10 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; +import java.io.*; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; +import java.util.*; import java.util.concurrent.TimeUnit; /** @@ -370,7 +362,7 @@ public class FeedEvictorTest { } } - @Test(enabled = false) + @Test public void testEvictionWithEmptyDirs() throws Exception { try { Configuration conf = cluster.getConf(); @@ -398,7 +390,7 @@ public class FeedEvictorTest { compare(map.get("feed1"), stream.getBuffer()); - String expectedInstancePaths = getExpectedInstancePaths(dataPath.replaceAll(storageUrl, "")); + String expectedInstancePaths = getExpectedInstancePaths(dataPath); Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths); String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1]; @@ -421,6 +413,49 @@ public class FeedEvictorTest { } } + @Test + public void testFeedBasePathExists() throws Exception { + try { + Configuration conf = cluster.getConf(); + FileSystem fs = FileSystem.get(conf); + fs.delete(new Path("/"), true); + stream.clear(); + + Pair<List<String>, List<String>> pair = generateInstances(fs, "feed3", + "yyyy/MM/dd/hh/mm", 1, TimeUnit.MINUTES, "/data", false); + final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY); + String dataPath = LocationType.DATA.name() + "=" + + storageUrl + "/data/YYYY/feed3/mmHH/dd/MM/?{YEAR}/?{MONTH}/?{DAY}/?{HOUR}/?{MINUTE}"; + String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv"; + long beforeDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount(); + + FeedEvictor.main(new String[]{ + "-feedBasePath", dataPath, + "-retentionType", "instance", + "-retentionLimit", "minutes(-1)", + "-timeZone", "UTC", + "-frequency", "minutes", + "-logFile", logFile, + "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(), + }); + + String expectedInstancePaths = getExpectedInstancePaths(dataPath); + Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths); + + //Feed Base path must exist + Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed3/mmHH/dd/MM/"))); + FileStatus[] files = fs.listStatus(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))); + //Number of directories/files inside feed base path should be 0 + Assert.assertEquals(files.length, 0); + long afterDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount(); + //Number of directories deleted + Assert.assertEquals((beforeDelCount - afterDelCount), 11); + + } catch (Exception e) { + Assert.fail("Unknown exception", e); + } + } + private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception { Configuration conf = cluster.getConf();
