Repository: falcon Updated Branches: refs/heads/master 65c804cba -> 7183416a9
FALCON-2023 Feed eviction fails when feed locations stats and meta ⦠â¦does not have time pattern Author: Venkatesan Ramachandran <[email protected]> Reviewers: " sandeepSamudrala <[email protected]>, Ying Zheng <[email protected]>, Balu Vellanki <[email protected]>" Closes #182 from vramachan/FALCON-2023 (cherry picked from commit df6b53c83520c9cd015dfbd0d752f25d18cd9c3f) Signed-off-by: bvellanki <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7183416a Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7183416a Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7183416a Branch: refs/heads/master Commit: 7183416a92a9f7c8285f12b997b6eacd9e6a9710 Parents: 65c804c Author: Venkatesan Ramachandran <[email protected]> Authored: Mon Jun 13 15:19:05 2016 -0700 Committer: bvellanki <[email protected]> Committed: Mon Jun 13 15:19:36 2016 -0700 ---------------------------------------------------------------------- .../apache/falcon/entity/FileSystemStorage.java | 11 +++- .../falcon/retention/FeedEvictorTest.java | 53 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/7183416a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java index ece8b5d..eb15585 100644 --- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java +++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java @@ -358,13 +358,20 @@ public class FileSystemStorage extends Configured implements Storage { private FileStatus[] findFilesForFeed(FileSystem fs, String feedBasePath) throws IOException { Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath); + boolean regexMatchFound = false; while (matcher.find()) { + regexMatchFound = true; String var = feedBasePath.substring(matcher.start(), matcher.end()); feedBasePath = feedBasePath.replaceAll(Pattern.quote(var), "*"); matcher = FeedDataPath.PATTERN.matcher(feedBasePath); } - LOG.info("Searching for {}", feedBasePath); - return fs.globStatus(new Path(feedBasePath)); + if (regexMatchFound) { + LOG.info("Searching for {}", feedBasePath); + return fs.globStatus(new Path(feedBasePath)); + } else { + LOG.info("Ignoring static path {}", feedBasePath); + return null; + } } private boolean isDateInRange(Date date, Date start) { http://git-wip-us.apache.org/repos/asf/falcon/blob/7183416a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java ---------------------------------------------------------------------- diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java index 72447da..98936ae 100644 --- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java +++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java @@ -462,6 +462,43 @@ public class FeedEvictorTest { } } + @Test + public void testEvictionStatsMetaWithNoPattern() throws Exception { + try { + Configuration conf = cluster.getConf(); + FileSystem fs = FileSystem.get(conf); + fs.delete(new Path("/"), true); + stream.clear(); + + Pair<List<String>, List<String>> pair = createTestData("/data"); + createDir("/stats"); + createDir("/meta"); + createTestData("/tmp"); + final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY); + FeedEvictor.main(new String[] { + "-feedBasePath", + getFeedBasePath(LocationType.DATA, storageUrl) + "#" + + getStatsOrMetaPath(LocationType.STATS, storageUrl) + + "#" + getStatsOrMetaPath(LocationType.META, storageUrl) + + "#" + getFeedBasePath(LocationType.TMP, storageUrl), + "-retentionType", "instance", + "-retentionLimit", "months(5)", + "-timeZone", "UTC", + "-frequency", "hourly", + "-logFile", conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY) + + "/falcon/staging/feed/2012-01-01-04-00", "-falconFeedStorageType", + Storage.TYPE.FILESYSTEM.name(), + }); + + // should not throw exception + // stats and meta dir should not be deleted + Assert.assertTrue(isDirPresent("/stats")); + Assert.assertTrue(isDirPresent("/meta")); + } catch (Exception e) { + Assert.fail("Unknown exception", e); + } + } + private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception { Configuration conf = cluster.getConf(); @@ -482,6 +519,12 @@ public class FeedEvictorTest { return Pair.of(inRange, outOfRange); } + private void createDir(String locationType) throws Exception { + Configuration conf = cluster.getConf(); + FileSystem fs = FileSystem.get(conf); + touch(fs, locationType, false); + } + private Pair<List<String>, List<String>> createTestData(String feed, String mask, int period, TimeUnit timeUnit, String locationType) throws Exception { @@ -542,11 +585,21 @@ public class FeedEvictorTest { } } + private boolean isDirPresent(String path) throws Exception { + FileSystem fs = FileSystem.get(cluster.getConf()); + return fs.exists(new Path(path)); + } + private String getFeedBasePath(LocationType locationType, String storageUrl) { return locationType.name() + "=" + storageUrl + "/" + locationType.name().toLowerCase() + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}"; } + private String getStatsOrMetaPath(LocationType locationType, String storageUrl) { + return locationType.name() + "=" + storageUrl + + "/" + locationType.name().toLowerCase(); + } + private static class InMemoryWriter extends PrintStream { private final StringBuffer buffer = new StringBuffer();
