Repository: falcon Updated Branches: refs/heads/master c2f7a2a5e -> 820572d8a
FALCON-1840 Archive older definition in case of update Author: Praveen Adlakha <[email protected]> Reviewers: Peeyush Bishnoi<[email protected]>, Balu<[email protected]>, Pallavi<[email protected]>, Sandeep<[email protected]> Closes #75 from PraveenAdlakha/1840 and squashes the following commits: c9c19d8 [Praveen Adlakha] documentation added for archival in delete and update 6c570e3 [Praveen Adlakha] comments addressed b475fc4 [Praveen Adlakha] IT added 7ba81af [Praveen Adlakha] FALCON-1840 Archive older definition in case of update Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/820572d8 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/820572d8 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/820572d8 Branch: refs/heads/master Commit: 820572d8afb5b1244af7f8c36c810c4efd0b749d Parents: c2f7a2a Author: Praveen Adlakha <[email protected]> Authored: Wed Mar 23 20:43:12 2016 +0530 Committer: peeyush b <[email protected]> Committed: Wed Mar 23 20:43:12 2016 +0530 ---------------------------------------------------------------------- .../falcon/entity/store/ConfigurationStore.java | 15 ++++- .../entity/store/FeedLocationStoreTest.java | 58 ++++++++++++++++++-- .../src/site/twiki/falconcli/DeleteEntity.twiki | 2 +- .../src/site/twiki/falconcli/UpdateEntity.twiki | 3 +- 4 files changed, 68 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java index 9c7a932..bdcd1af 100644 --- a/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java +++ b/common/src/main/java/org/apache/falcon/entity/store/ConfigurationStore.java @@ -98,10 +98,18 @@ public final class ConfigurationStore implements FalconService { public static ConfigurationStore get() { return STORE; } - private FileSystem fs; private Path storePath; + public FileSystem getFs() { + return fs; + } + + public Path getStorePath() { + return storePath; + } + + private ConfigurationStore() { for (EntityType type : EntityType.values()) { dictionary.put(type, new ConcurrentHashMap<String, Entity>()); @@ -250,6 +258,11 @@ public final class ConfigurationStore implements FalconService { public synchronized void update(EntityType type, Entity entity) throws FalconException { if (updatesInProgress.get() == entity) { + try { + archive(type, entity.getName()); + } catch (IOException e) { + throw new StoreAccessException(e); + } updateInternal(type, entity); } else { throw new FalconException(entity.toShortString() + " is not initialized for update"); http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java index 033a55b..40c077e 100644 --- a/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java +++ b/common/src/test/java/org/apache/falcon/entity/store/FeedLocationStoreTest.java @@ -33,6 +33,8 @@ import org.apache.falcon.security.CurrentUser; import org.apache.falcon.util.FalconRadixUtils; import org.apache.falcon.util.FalconTestUtil; import org.apache.falcon.util.StartupProperties; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; @@ -40,6 +42,7 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; +import java.io.IOException; import java.net.URI; import java.util.Collection; @@ -50,7 +53,6 @@ import java.util.Collection; public class FeedLocationStoreTest extends AbstractTestBase { private ConfigurationStore store; - @BeforeClass public void initConfigStore() throws Exception { String configPath = new URI(StartupProperties.get().getProperty("config.store.uri")).getPath(); @@ -65,7 +67,6 @@ public class FeedLocationStoreTest extends AbstractTestBase { StartupProperties.get().setProperty("configstore.listeners", listeners); store = ConfigurationStore.get(); store.init(); - CurrentUser.authenticate(FalconTestUtil.TEST_USER_2); } @@ -102,10 +103,41 @@ public class FeedLocationStoreTest extends AbstractTestBase { } @Test + public void testOnUpdate() throws FalconException{ + Feed f1 = createFeed("f1"); + f1.getLocations().getLocations().add(createLocation(LocationType.DATA, + "/projects/cas/data/hourly/2014/09/09/09")); + store.publish(EntityType.FEED, f1); + + Feed f2 = createFeed("f1"); + f2.getLocations().getLocations().add(createLocation(LocationType.DATA, + "/projects/cas/data/monthly")); + store.initiateUpdate(f2); + store.update(EntityType.FEED, f2); + store.cleanupUpdateInit(); + boolean isArchived = false; + try { + Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR + "FEED"); + FileStatus [] files= store.getFs().listStatus(archivePath); + for(FileStatus f:files){ + String name = f.getPath().getName(); + if (name.startsWith(f2.getName())){ + isArchived= true; + break; + } + } + }catch (IOException e){ + e.printStackTrace(); + } + Assert.assertTrue(isArchived); + } + + + @Test public void testOnRemove() throws FalconException{ int initialSize = FeedLocationStore.get().store.getSize(); - - Feed f1 = createFeed("f1ForRemove"); + String feedName = "f1ForRemove"; + Feed f1 = createFeed(feedName); f1.getLocations().getLocations().add(createLocation(LocationType.DATA, "/projects/cas/data/hourly/2014/09/09/09")); f1.getLocations().getLocations().add(createLocation(LocationType.STATS, @@ -113,9 +145,23 @@ public class FeedLocationStoreTest extends AbstractTestBase { store.publish(EntityType.FEED, f1); Assert.assertEquals(FeedLocationStore.get().store.getSize() - initialSize, 4); - store.remove(EntityType.FEED, "f1ForRemove"); + store.remove(EntityType.FEED, feedName); + boolean isArchived = false; + try { + Path archivePath = new Path(store.getStorePath(), "archive" + Path.SEPARATOR + "FEED"); + FileStatus [] files= store.getFs().listStatus(archivePath); + for(FileStatus f:files){ + String name = f.getPath().getName(); + if (name.startsWith(feedName)){ + isArchived= true; + break; + } + } + }catch (IOException e){ + e.printStackTrace(); + } + Assert.assertTrue(isArchived); Assert.assertEquals(FeedLocationStore.get().store.getSize(), initialSize); - } http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/DeleteEntity.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/falconcli/DeleteEntity.twiki b/docs/src/site/twiki/falconcli/DeleteEntity.twiki index f2b3080..cc07406 100644 --- a/docs/src/site/twiki/falconcli/DeleteEntity.twiki +++ b/docs/src/site/twiki/falconcli/DeleteEntity.twiki @@ -2,7 +2,7 @@ [[CommonCLI][Common CLI Options]] -Delete removes the submitted entity definition for the specified entity and put it into the archive. +Delete removes the submitted entity definition for the specified entity and put it into the archive.Archive path is defined in startup.properties in variable "config.store.uri". Usage: $FALCON_HOME/bin/falcon entity -type [cluster|datasource|feed|process] -name <<name>> -delete http://git-wip-us.apache.org/repos/asf/falcon/blob/820572d8/docs/src/site/twiki/falconcli/UpdateEntity.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/falconcli/UpdateEntity.twiki b/docs/src/site/twiki/falconcli/UpdateEntity.twiki index ae60559..5d49a76 100644 --- a/docs/src/site/twiki/falconcli/UpdateEntity.twiki +++ b/docs/src/site/twiki/falconcli/UpdateEntity.twiki @@ -2,8 +2,7 @@ [[CommonCLI][Common CLI Options]] -Update operation allows an already submitted/scheduled entity to be updated. Cluster and datasource updates are -currently not allowed. +Update operation allows an already submitted/scheduled entity to be updated and put it into the archive.Archive path is defined in startup.properties in variable "config.store.uri". Cluster and datasource updates are currently not allowed. Usage: $FALCON_HOME/bin/falcon entity -type [feed|process] -name <<name>> -update -file <<path_to_file>>
