FALCON-968 Remove only entities submitted by same test and baseHDFSDir usage refactored. Contributed by Ruslan Ostafiychuk
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/0692b241 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/0692b241 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/0692b241 Branch: refs/heads/master Commit: 0692b2415ebf9ae5ad6a07aae32532a1ed01ec43 Parents: e422fce Author: Raghav Kumar Gautam <[email protected]> Authored: Wed Feb 4 12:47:24 2015 -0800 Committer: Raghav Kumar Gautam <[email protected]> Committed: Wed Feb 4 12:47:24 2015 -0800 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 4 + .../regression/Entities/ClusterMerlin.java | 6 +- .../falcon/regression/Entities/FeedMerlin.java | 6 +- .../regression/Entities/ProcessMerlin.java | 6 +- .../regression/Entities/TestEntityUtil.java | 7 ++ .../falcon/regression/core/bundle/Bundle.java | 28 +++-- .../regression/core/util/CleanupUtil.java | 110 ++++++------------- .../falcon/regression/core/util/Util.java | 8 -- .../regression/testHelper/BaseTestClass.java | 27 +++-- .../falcon/regression/AuthorizationTest.java | 14 +-- .../regression/ELExpFutureAndLatestTest.java | 8 +- .../falcon/regression/ELValidationsTest.java | 4 +- .../regression/EmbeddedPigScriptTest.java | 10 +- .../falcon/regression/ExternalFSTest.java | 15 +-- .../regression/FeedClusterUpdateTest.java | 10 +- .../regression/FeedInstanceStatusTest.java | 6 +- .../falcon/regression/FeedLateRerunTest.java | 13 +-- .../falcon/regression/FeedReplicationTest.java | 10 +- .../falcon/regression/FeedResumeTest.java | 4 +- .../falcon/regression/FeedScheduleTest.java | 4 +- .../falcon/regression/FeedStatusTest.java | 4 +- .../regression/FeedSubmitAndScheduleTest.java | 4 +- .../falcon/regression/FeedSubmitTest.java | 4 +- .../falcon/regression/FeedSuspendTest.java | 4 +- .../falcon/regression/InstanceParamTest.java | 8 +- .../falcon/regression/InstanceSummaryTest.java | 8 +- .../apache/falcon/regression/NewRetryTest.java | 6 +- .../falcon/regression/NoOutputProcessTest.java | 10 +- .../falcon/regression/ProcessFrequencyTest.java | 8 +- .../ProcessInstanceColoMixedTest.java | 8 +- .../regression/ProcessInstanceKillsTest.java | 9 +- .../regression/ProcessInstanceRerunTest.java | 6 +- .../regression/ProcessInstanceResumeTest.java | 6 +- .../regression/ProcessInstanceRunningTest.java | 8 +- .../regression/ProcessInstanceStatusTest.java | 8 +- .../regression/ProcessInstanceSuspendTest.java | 6 +- .../falcon/regression/ProcessLateRerunTest.java | 6 +- .../falcon/regression/ProcessLibPathTest.java | 12 +- .../regression/entity/ListEntitiesTest.java | 20 ++-- .../regression/hcat/HCatFeedOperationsTest.java | 8 +- .../falcon/regression/hcat/HCatProcessTest.java | 15 ++- .../regression/hcat/HCatReplicationTest.java | 10 +- .../regression/hcat/HCatRetentionTest.java | 4 +- .../regression/lineage/EntitySummaryTest.java | 8 +- .../lineage/LineageApiProcessInstanceTest.java | 6 +- .../regression/lineage/LineageApiTest.java | 21 ++-- .../lineage/ListFeedInstancesTest.java | 7 +- .../lineage/ListProcessInstancesTest.java | 6 +- .../regression/lineage/ProcessPipelineTest.java | 62 +++++------ .../regression/prism/EntityDryRunTest.java | 6 +- .../prism/FeedDelayParallelTimeoutTest.java | 8 +- .../prism/NewPrismProcessUpdateTest.java | 20 ++-- .../regression/prism/OptionalInputTest.java | 6 +- .../prism/PrismClusterDeleteTest.java | 6 +- .../prism/PrismConcurrentRequestTest.java | 6 +- .../regression/prism/PrismFeedDeleteTest.java | 21 ++-- .../prism/PrismFeedLateReplicationTest.java | 6 +- .../PrismFeedReplicationPartitionExpTest.java | 6 +- .../prism/PrismFeedReplicationUpdateTest.java | 10 +- .../regression/prism/PrismFeedResumeTest.java | 6 +- .../regression/prism/PrismFeedScheduleTest.java | 6 +- .../regression/prism/PrismFeedSnSTest.java | 11 +- .../regression/prism/PrismFeedSuspendTest.java | 6 +- .../regression/prism/PrismFeedUpdateTest.java | 11 +- .../prism/PrismProcessDeleteTest.java | 6 +- .../prism/PrismProcessResumeTest.java | 6 +- .../prism/PrismProcessScheduleTest.java | 9 +- .../regression/prism/PrismProcessSnSTest.java | 6 +- .../prism/PrismProcessSuspendTest.java | 6 +- .../regression/prism/PrismSubmitTest.java | 8 +- .../prism/ProcessPartitionExpVariableTest.java | 6 +- .../prism/RescheduleKilledProcessTest.java | 12 +- .../RescheduleProcessInFinalStatesTest.java | 8 +- .../falcon/regression/prism/RetentionTest.java | 8 +- .../prism/UpdateAtSpecificTimeTest.java | 14 +-- .../regression/security/AclValidationTest.java | 6 +- .../regression/security/ClusterAclTest.java | 6 +- .../falcon/regression/security/EntityOp.java | 19 +--- .../regression/security/FalconClientTest.java | 6 +- .../falcon/regression/security/FeedAclTest.java | 6 +- .../regression/security/ProcessAclTest.java | 6 +- .../falcon/regression/ui/LineageGraphTest.java | 6 +- .../falcon/regression/ui/ProcessUITest.java | 24 ++-- 83 files changed, 420 insertions(+), 475 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 687793e..404920f 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -39,6 +39,10 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS + + FALCON-968 Remove only entities submitted by same test and baseHDFSDir usage refactored + (Ruslan Ostafiychuk via Raghav Kumar Gautam) + FALCON-1007 Improve test output and variable names (Paul Isaychuk via Ruslan Ostafiychuk) FALCON-1009 InstanceUtil cleanup (Raghav Kumar Gautam) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java index b1a94be..22d8ca4 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java @@ -23,7 +23,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.ACL; import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.regression.core.util.Util; import org.testng.Assert; import javax.xml.bind.JAXBException; @@ -73,10 +72,11 @@ public class ClusterMerlin extends Cluster { /** * Sets unique names for the cluster. * @return mapping of old name to new name + * @param prefix prefix of new name */ - public Map<? extends String, ? extends String> setUniqueName() { + public Map<? extends String, ? extends String> setUniqueName(String prefix) { final String oldName = getName(); - final String newName = oldName + Util.getUniqueString(); + final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); setName(newName); final HashMap<String, String> nameMap = new HashMap<String, String>(1); nameMap.put(oldName, newName); http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java index cec7684..47eb1bd 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java @@ -36,7 +36,6 @@ import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.entity.v0.feed.Retention; import org.apache.falcon.entity.v0.feed.Validity; import org.apache.falcon.regression.core.util.TimeUtil; -import org.apache.falcon.regression.core.util.Util; import org.testng.Assert; import javax.xml.bind.JAXBException; @@ -213,10 +212,11 @@ public class FeedMerlin extends Feed { /** * Sets unique names for the feed. * @return mapping of old name to new name + * @param prefix prefix of new name */ - public Map<? extends String, ? extends String> setUniqueName() { + public Map<? extends String, ? extends String> setUniqueName(String prefix) { final String oldName = getName(); - final String newName = oldName + Util.getUniqueString(); + final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); setName(newName); final HashMap<String, String> nameMap = new HashMap<String, String>(1); nameMap.put(oldName, newName); http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java index 9ffc15d..36b0df5 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java @@ -34,7 +34,6 @@ import org.apache.falcon.entity.v0.process.Properties; import org.apache.falcon.entity.v0.process.Property; import org.apache.falcon.entity.v0.process.Validity; import org.apache.falcon.regression.core.util.TimeUtil; -import org.apache.falcon.regression.core.util.Util; import org.testng.Assert; import javax.xml.bind.JAXBException; @@ -175,10 +174,11 @@ public class ProcessMerlin extends Process { /** * Sets unique names for the process. * @return mapping of old name to new name + * @param prefix prefix of new name */ - public Map<? extends String, ? extends String> setUniqueName() { + public Map<? extends String, ? extends String> setUniqueName(String prefix) { final String oldName = getName(); - final String newName = oldName + Util.getUniqueString(); + final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); setName(newName); final HashMap<String, String> nameMap = new HashMap<String, String>(1); nameMap.put(oldName, newName); http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java index 0d3e0ae..d597073 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java @@ -20,9 +20,11 @@ package org.apache.falcon.regression.Entities; import org.apache.falcon.entity.v0.Entity; import org.apache.falcon.entity.v0.EntityType; +import org.testng.Assert; import javax.xml.bind.Unmarshaller; import java.io.StringReader; +import java.util.UUID; /** * Util class for merlin entities. @@ -43,4 +45,9 @@ final class TestEntityUtil { } } + public static String generateUniqueName(String prefix, String oldName) { + Assert.assertNotNull(prefix, "name prefix shouldn't be null!"); + return prefix + '-' + oldName + '-' + UUID.randomUUID().toString().split("-")[0]; + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java index 91abe89..8e449fb 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java @@ -218,18 +218,26 @@ public class Bundle { * Generates unique entities within a bundle changing their names and names of dependant items * to unique. */ - public void generateUniqueBundle() { + public void generateUniqueBundle(Object testClassObject) { + generateUniqueBundle(testClassObject.getClass().getSimpleName() + '-'); + } + + /** + * Generates unique entities within a bundle changing their names and names of dependant items + * to unique. + */ + public void generateUniqueBundle(String prefix) { /* creating new names */ List<ClusterMerlin> clusterMerlinList = ClusterMerlin.fromString(clusters); Map<String, String> clusterNameMap = new HashMap<String, String>(); for (ClusterMerlin clusterMerlin : clusterMerlinList) { - clusterNameMap.putAll(clusterMerlin.setUniqueName()); + clusterNameMap.putAll(clusterMerlin.setUniqueName(prefix)); } List<FeedMerlin> feedMerlinList = FeedMerlin.fromString(dataSets); Map<String, String> feedNameMap = new HashMap<String, String>(); for (FeedMerlin feedMerlin : feedMerlinList) { - feedNameMap.putAll(feedMerlin.setUniqueName()); + feedNameMap.putAll(feedMerlin.setUniqueName(prefix)); } /* setting new names in feeds and process */ @@ -249,7 +257,7 @@ public class Bundle { if (StringUtils.isNotEmpty(processData)) { ProcessMerlin processMerlin = new ProcessMerlin(processData); - processMerlin.setUniqueName(); + processMerlin.setUniqueName(prefix); processMerlin.renameClusters(clusterNameMap); processMerlin.renameFeeds(feedNameMap); processData = processMerlin.toString(); @@ -820,12 +828,11 @@ public class Bundle { * @param endTime end of feeds and process validity on every cluster */ public void generateRequiredBundle(int numberOfClusters, int numberOfInputs, - int numberOfOptionalInput, - String inputBasePaths, int numberOfOutputs, String startTime, - String endTime) { + int numberOfOptionalInput, + String inputBasePaths, int numberOfOutputs, String startTime, + String endTime) { //generate and set clusters ClusterMerlin c = new ClusterMerlin(getClusters().get(0)); - c.setUniqueName(); List<String> newClusters = new ArrayList<String>(); final String clusterName = c.getName(); for (int i = 0; i < numberOfClusters; i++) { @@ -838,13 +845,13 @@ public class Bundle { List<String> newDataSets = new ArrayList<String>(); for (int i = 0; i < numberOfInputs; i++) { final FeedMerlin feed = new FeedMerlin(getDataSets().get(0)); - feed.setUniqueName(); + feed.setName(feed.getName() + "-input" + i); feed.setFeedClusters(newClusters, inputBasePaths + "/input" + i, startTime, endTime); newDataSets.add(feed.toString()); } for (int i = 0; i < numberOfOutputs; i++) { final FeedMerlin feed = new FeedMerlin(getDataSets().get(0)); - feed.setUniqueName(); + feed.setName(feed.getName() + "-output" + i); feed.setFeedClusters(newClusters, inputBasePaths + "/output" + i, startTime, endTime); newDataSets.add(feed.toString()); } @@ -852,7 +859,6 @@ public class Bundle { //add clusters and feed to process ProcessMerlin processMerlin = new ProcessMerlin(getProcessData()); - processMerlin.setUniqueName(); processMerlin.setProcessClusters(newClusters, startTime, endTime); processMerlin.setProcessFeeds(newDataSets, numberOfInputs, numberOfOptionalInput, numberOfOutputs); http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java index e997fc8..89ae254 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/CleanupUtil.java @@ -44,36 +44,29 @@ public final class CleanupUtil { } private static final Logger LOGGER = Logger.getLogger(CleanupUtil.class); - public static List<String> getAllProcesses(ColoHelper prism) - throws IOException, URISyntaxException, AuthenticationException, JAXBException, - InterruptedException { - return getAllEntitiesOfOneType(prism.getProcessHelper(), null); - } - - public static List<String> getAllFeeds(ColoHelper prism) - throws IOException, URISyntaxException, AuthenticationException, JAXBException, - InterruptedException { - return getAllEntitiesOfOneType(prism.getFeedHelper(), null); - } - - public static List<String> getAllClusters(ColoHelper prism) - throws IOException, URISyntaxException, AuthenticationException, JAXBException, - InterruptedException { - return getAllEntitiesOfOneType(prism.getClusterHelper(), null); + public static List<String> getAllEntitiesOfOneType(AbstractEntityHelper entityManagerHelper, + String user) { + return getEntitiesWithPrefix(entityManagerHelper, user, ""); } - public static List<String> getAllEntitiesOfOneType(AbstractEntityHelper entityManagerHelper, - String user) - throws IOException, URISyntaxException, AuthenticationException, JAXBException, - InterruptedException { - final EntityList entityList = getEntitiesResultOfOneType(entityManagerHelper, user); - List<String> clusters = new ArrayList<String>(); + public static List<String> getEntitiesWithPrefix(AbstractEntityHelper entityHelper, + String user, String namePrefix) { + final EntityList entityList; + try { + entityList = getEntitiesResultOfOneType(entityHelper, user); + } catch (Exception e) { + LOGGER.error("Caught exception: " + ExceptionUtils.getStackTrace(e)); + return null; + } + List<String> entities = new ArrayList<String>(); if (entityList.getElements() != null) { for (EntityList.EntityElement entity : entityList.getElements()) { - clusters.add(entity.name); + if (entity.name.startsWith(namePrefix)) { + entities.add(entity.name); + } } } - return clusters; + return entities; } private static EntityList getEntitiesResultOfOneType( @@ -87,67 +80,28 @@ public final class CleanupUtil { new StringReader(clusterResponse.getMessage())); } - public static void cleanAllClustersQuietly(ColoHelper prism) { - try { - final List<String> clusters = getAllClusters(prism); - for (String cluster : clusters) { - try { - prism.getClusterHelper().deleteByName(cluster, null); - } catch (Exception e) { - LOGGER.warn("Caught exception: " + ExceptionUtils.getStackTrace(e)); - } - } - } catch (Exception e) { - LOGGER.warn("Unable to get a list of clusters because of exception: " - + - ExceptionUtils.getStackTrace(e)); - } - } - public static void cleanAllFeedsQuietly(ColoHelper prism) { - try { - final List<String> feeds = getAllFeeds(prism); - for (String feed : feeds) { - try { - prism.getFeedHelper().deleteByName(feed, null); - } catch (Exception e) { - LOGGER.warn("Caught exception: " + ExceptionUtils.getStackTrace(e)); - } - } - } catch (Exception e) { - LOGGER.warn("Unable to get a list of feeds because of exception: " - + - ExceptionUtils.getStackTrace(e)); - } - } - public static void cleanAllProcessesQuietly(ColoHelper prism, - AbstractEntityHelper entityManagerHelper) { - try { - final List<String> processes = getAllProcesses(prism); - for (String process : processes) { - try { - entityManagerHelper.deleteByName(process, null); - } catch (Exception e) { - LOGGER.warn("Caught exception: " + ExceptionUtils.getStackTrace(e)); - } - } - } catch (Exception e) { - LOGGER.warn("Unable to get a list of feeds because of exception: " - + - ExceptionUtils.getStackTrace(e)); + public static void cleanEntitiesWithPrefix(ColoHelper prism, String namePrefix) { + final List<String> processes = getEntitiesWithPrefix(prism.getProcessHelper(), null, namePrefix); + final List<String> feeds = getEntitiesWithPrefix(prism.getFeedHelper(), null, namePrefix); + final List<String> clusters = getEntitiesWithPrefix(prism.getClusterHelper(), null, namePrefix); + + for (String process : processes) { + deleteQuietlyByName(prism.getProcessHelper(), process); + } + for (String feed : feeds) { + deleteQuietlyByName(prism.getFeedHelper(), feed); } - } - public static void cleanAllEntities(ColoHelper prism) { - cleanAllProcessesQuietly(prism, prism.getProcessHelper()); - cleanAllFeedsQuietly(prism); - cleanAllClustersQuietly(prism); + for (String cluster : clusters) { + deleteQuietlyByName(prism.getClusterHelper(), cluster); + } } - public static void deleteQuietly(AbstractEntityHelper helper, String feed) { + private static void deleteQuietlyByName(AbstractEntityHelper helper, String entityName) { try { - helper.delete(feed); + helper.deleteByName(entityName, null); } catch (Exception e) { LOGGER.info("Caught exception: " + ExceptionUtils.getStackTrace(e)); } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java index 58e3691..5717923 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java @@ -78,7 +78,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; -import java.util.UUID; /** * util methods used across test. @@ -198,13 +197,6 @@ public final class Util { } /** - * @return unique string - */ - public static String getUniqueString() { - return "-" + UUID.randomUUID().toString().split("-")[0]; - } - - /** * Retrieves all hadoop data directories from a specific data path. * @param fs filesystem * @param feed feed definition http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java index a282ee0..a3bd096 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/testHelper/BaseTestClass.java @@ -21,6 +21,7 @@ package org.apache.falcon.regression.testHelper; import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.util.CleanupUtil; import org.apache.falcon.regression.core.util.Config; import org.apache.falcon.regression.core.util.HadoopUtil; import org.apache.hadoop.fs.FileSystem; @@ -46,7 +47,7 @@ public class BaseTestClass { protected List<ColoHelper> servers; protected List<FileSystem> serverFS; protected List<OozieClient> serverOC; - protected String baseHDFSDir = "/tmp/falcon-regression"; + private String baseHDFSDir = "/tmp/falcon-regression"; public static final String PRISM_PREFIX = "prism"; protected Bundle[] bundles; public static final String MINUTE_DATE_PATTERN = "/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"; @@ -69,6 +70,17 @@ public class BaseTestClass { } bundles = new Bundle[serverNames.length]; + removeTestClassEntities(); + } + + protected final String cleanAndGetTestDir() { + String dir = baseHDFSDir + '/' + this.getClass().getSimpleName(); + try { + HadoopUtil.recreateDir(serverFS, dir); + } catch (IOException e) { + throw new RuntimeException(e); + } + return dir; } private static void prepareProperties() { @@ -96,21 +108,16 @@ public class BaseTestClass { } } - public final void removeBundles(Bundle... bundlesForDeletion) { - for (Bundle bundle : bundlesForDeletion) { + public final void removeTestClassEntities() { + for (Bundle bundle : this.bundles) { if (bundle != null) { bundle.deleteBundle(prism); } } - if (bundlesForDeletion != this.bundles) { - for (Bundle bundle : this.bundles) { - if (bundle != null) { - bundle.deleteBundle(prism); - } - } - } + CleanupUtil.cleanEntitiesWithPrefix(prism, this.getClass().getSimpleName()); } + public final void cleanTestsDirs() throws IOException { if (MerlinConstants.CLEAN_TESTS_DIR) { for (FileSystem fs : serverFS) { http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java index ad697fc..4787ef1 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java @@ -65,7 +65,7 @@ public class AuthorizationTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String baseTestDir = baseHDFSDir + "/AuthorizationTest"; + private String baseTestDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestDir + "/aggregator"; private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; @@ -78,7 +78,7 @@ public class AuthorizationTest extends BaseTestClass { public void setup() throws Exception { Bundle bundle = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundle, cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setProcessWorkflow(aggregateWorkflowDir); } @@ -509,7 +509,7 @@ public class AuthorizationTest extends BaseTestClass { .readEntityName(feed)) && !definition.contains("(feed) not found"), "Feed should be already submitted"); //update feed definition - String newFeed = Util.setFeedPathValue(feed, baseHDFSDir + "/randomPath" + String newFeed = Util.setFeedPathValue(feed, baseTestDir + "/randomPath" + MINUTE_DATE_PATTERN); //try to update feed by U2 final ServiceResponse serviceResponse = prism.getFeedHelper().update(feed, newFeed, @@ -529,7 +529,7 @@ public class AuthorizationTest extends BaseTestClass { AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed)); AssertUtil.checkStatus(clusterOC, EntityType.FEED, feed, Job.Status.RUNNING); //update feed definition - String newFeed = Util.setFeedPathValue(feed, baseHDFSDir + "/randomPath" + String newFeed = Util.setFeedPathValue(feed, baseTestDir + "/randomPath" + MINUTE_DATE_PATTERN); //try to update feed by U2 final ServiceResponse serviceResponse = prism.getFeedHelper().update(feed, newFeed, @@ -614,7 +614,7 @@ public class AuthorizationTest extends BaseTestClass { .getLatestBundleID(cluster, Util.readEntityName(feed), EntityType.FEED); //update feed definition - String newFeed = Util.setFeedPathValue(feed, baseHDFSDir + "/randomPath" + String newFeed = Util.setFeedPathValue(feed, baseTestDir + "/randomPath" + MINUTE_DATE_PATTERN); //update feed by U1 @@ -652,7 +652,7 @@ public class AuthorizationTest extends BaseTestClass { AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, process, Job.Status.RUNNING); //update feed definition - String newFeed = Util.setFeedPathValue(feed, baseHDFSDir + "/randomPath" + String newFeed = Util.setFeedPathValue(feed, baseTestDir + "/randomPath" + MINUTE_DATE_PATTERN); //update feed by U2 @@ -680,6 +680,6 @@ public class AuthorizationTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java index aea788a..6939973 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java @@ -48,7 +48,7 @@ public class ELExpFutureAndLatestTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String baseTestDir = baseHDFSDir + "/ELExpFutureAndLatestTest"; + private String baseTestDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestDir + "/aggregator"; private static final Logger LOGGER = Logger.getLogger(ELExpFutureAndLatestTest.class); @@ -58,7 +58,7 @@ public class ELExpFutureAndLatestTest extends BaseTestClass { uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle b = BundleUtil.readELBundle(); - b.generateUniqueBundle(); + b.generateUniqueBundle(this); b = new Bundle(b, cluster); String startDate = TimeUtil.getTimeWrtSystemTime(-20); @@ -77,7 +77,7 @@ public class ELExpFutureAndLatestTest extends BaseTestClass { public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(baseTestDir + "/testData" + MINUTE_DATE_PATTERN); bundles[0].setOutputFeedLocationData(baseTestDir + "/output" @@ -94,7 +94,7 @@ public class ELExpFutureAndLatestTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(groups = {"singleCluster"}) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java index 1306dc3..18efe9b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java @@ -57,7 +57,7 @@ public class ELValidationsTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private static final Logger LOGGER = Logger.getLogger(ELValidationsTest.class); - private String aggregateWorkflowDir = baseHDFSDir + "/ELTest/aggregator"; + private String aggregateWorkflowDir = cleanAndGetTestDir() + "/aggregator"; @Test(groups = {"0.1", "0.2"}) @@ -124,7 +124,7 @@ public class ELValidationsTest extends BaseTestClass { aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle bundle = BundleUtil.readELBundle(); bundle = new Bundle(bundle, cluster.getPrefix()); - bundle.generateUniqueBundle(); + bundle.generateUniqueBundle(this); bundle.setProcessWorkflow(aggregateWorkflowDir); if (feedStart != null && feedEnd != null) { bundle.setFeedValidity(feedStart, feedEnd, bundle.getInputFeedNameFromBundle()); http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java index c0113cd..bddf4ef 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java @@ -59,8 +59,8 @@ public class EmbeddedPigScriptTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String pigTestDir = baseHDFSDir + "/EmbeddedPigScriptTest"; - private String pigScriptDir = pigTestDir + "/EmbeddedPigScriptTest/pig"; + private String pigTestDir = cleanAndGetTestDir(); + private String pigScriptDir = pigTestDir + "/pig"; private String pigScriptLocation = pigScriptDir + "/id.pig"; private String inputPath = pigTestDir + "/input" + MINUTE_DATE_PATTERN; private static final Logger LOGGER = Logger.getLogger(EmbeddedPigScriptTest.class); @@ -75,7 +75,7 @@ public class EmbeddedPigScriptTest extends BaseTestClass { //copy pig script HadoopUtil.uploadDir(clusterFS, pigScriptDir, OSUtil.RESOURCES + "pig"); Bundle bundle = BundleUtil.readELBundle(); - bundle.generateUniqueBundle(); + bundle.generateUniqueBundle(this); bundle = new Bundle(bundle, cluster); String startDate = "2010-01-02T00:40Z"; String endDate = "2010-01-02T01:10Z"; @@ -89,7 +89,7 @@ public class EmbeddedPigScriptTest extends BaseTestClass { public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(inputPath); bundles[0].setOutputFeedLocationData(pigTestDir + "/output-data" + MINUTE_DATE_PATTERN); bundles[0].setProcessWorkflow(pigScriptLocation); @@ -115,7 +115,7 @@ public class EmbeddedPigScriptTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(groups = {"singleCluster"}, timeOut = 600000) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java index bb63985..6b227d6 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java @@ -58,6 +58,7 @@ import org.testng.annotations.Test; import javax.xml.bind.JAXBException; import java.io.IOException; import java.util.List; +import java.util.UUID; /** * Tests for operations with external file systems. @@ -73,11 +74,11 @@ public class ExternalFSTest extends BaseTestClass{ private FileSystem wasbFS; private Bundle externalBundle; - - private String baseTestDir = baseHDFSDir + "/ExternalFSTest"; + private String baseTestDir = cleanAndGetTestDir(); private String sourcePath = baseTestDir + "/source"; - private String baseWasbDir = "/falcon-regression/" + Util.getUniqueString().substring(1); - private String testWasbTargetDir = baseWasbDir + "/"+ Util.getUniqueString().substring(1) + "/"; + private String baseWasbDir = "/falcon-regression/" + UUID.randomUUID().toString().split("-")[0]; + private String testWasbTargetDir = baseWasbDir + '/' + + UUID.randomUUID().toString().split("-")[0] + '/'; private static final Logger LOGGER = Logger.getLogger(ExternalFSTest.class); @@ -99,8 +100,8 @@ public class ExternalFSTest extends BaseTestClass{ bundles[0] = new Bundle(bundle, cluster); externalBundle = new Bundle(bundle, cluster); - bundles[0].generateUniqueBundle(); - externalBundle.generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); + externalBundle.generateUniqueBundle(this); LOGGER.info("checking wasb credentials with location: " + testWasbTargetDir); wasbFS.create(new Path(testWasbTargetDir)); @@ -109,7 +110,7 @@ public class ExternalFSTest extends BaseTestClass{ @AfterMethod public void tearDown() throws IOException { - removeBundles(externalBundle); + removeTestClassEntities(); wasbFS.delete(new Path(testWasbTargetDir), true); } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java index a24a0ee..dab7335 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedClusterUpdateTest.java @@ -48,7 +48,7 @@ import org.testng.annotations.Test; @Test(groups = "distributed") public class FeedClusterUpdateTest extends BaseTestClass { - private String baseTestDir = baseHDFSDir + "/FeedClusterUpdateTest"; + private String baseTestDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestDir + "/aggregator"; private ColoHelper cluster1 = servers.get(0); private ColoHelper cluster2 = servers.get(1); @@ -68,7 +68,7 @@ public class FeedClusterUpdateTest extends BaseTestClass { Bundle bundle = BundleUtil.readELBundle(); for (int i = 0; i < 3; i++) { bundles[i] = new Bundle(bundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } try { @@ -79,7 +79,7 @@ public class FeedClusterUpdateTest extends BaseTestClass { HadoopUtil.deleteDirIfExists(baseTestDir, cluster3FS); HadoopUtil.lateDataReplenish(cluster3FS, 80, 1, baseTestDir, postFix); } finally { - removeBundles(); + removeTestClassEntities(); } } @@ -88,7 +88,7 @@ public class FeedClusterUpdateTest extends BaseTestClass { Bundle bundle = BundleUtil.readELBundle(); for (int i = 0; i < 3; i++) { bundles[i] = new Bundle(bundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } BundleUtil.submitAllClusters(prism, bundles[0], bundles[1], bundles[2]); @@ -99,7 +99,7 @@ public class FeedClusterUpdateTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(enabled = false, groups = {"multiCluster"}) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java index 5929117..514fd10 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java @@ -46,7 +46,7 @@ import org.testng.annotations.Test; @Test(groups = "embedded") public class FeedInstanceStatusTest extends BaseTestClass { - private String baseTestDir = baseHDFSDir + "/FeedInstanceStatusTest"; + private String baseTestDir = cleanAndGetTestDir(); private String feedInputPath = baseTestDir + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestDir + "/aggregator"; @@ -66,14 +66,14 @@ public class FeedInstanceStatusTest extends BaseTestClass { Bundle bundle = BundleUtil.readELBundle(); for (int i = 0; i < 3; i++) { bundles[i] = new Bundle(bundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java index 569f380..9751003 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java @@ -55,11 +55,10 @@ public class FeedLateRerunTest extends BaseTestClass { private FileSystem cluster1FS = serverFS.get(0); private FileSystem cluster2FS = serverFS.get(1); private OozieClient cluster2OC = serverOC.get(1); - private String dateTemplate = "/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}"; - private String baseTestDir = baseHDFSDir + "/FeedLateRerunTest"; - private String feedDataLocation = baseTestDir + "/source" + dateTemplate; + private String baseTestDir = cleanAndGetTestDir(); + private String feedDataLocation = baseTestDir + "/source" + MINUTE_DATE_PATTERN; private String targetPath = baseTestDir + "/target"; - private String targetDataLocation = targetPath + dateTemplate; + private String targetDataLocation = targetPath + MINUTE_DATE_PATTERN; private static final Logger LOGGER = Logger.getLogger(FeedLateRerunTest.class); private String source = null; private String target = null; @@ -69,13 +68,13 @@ public class FeedLateRerunTest extends BaseTestClass { Bundle bundle = BundleUtil.readFeedReplicationBundle(); bundles[0] = new Bundle(bundle, cluster1); bundles[1] = new Bundle(bundle, cluster2); - bundles[0].generateUniqueBundle(); - bundles[1].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); + bundles[1].generateUniqueBundle(this); } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(dataProvider = "dataFlagProvider") http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index 35842e4..b88cffc 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -68,7 +68,7 @@ public class FeedReplicationTest extends BaseTestClass { private FileSystem cluster3FS = serverFS.get(2); private OozieClient cluster2OC = serverOC.get(1); private OozieClient cluster3OC = serverOC.get(2); - private String baseTestDir = baseHDFSDir + "/FeedReplicationTest"; + private String baseTestDir = cleanAndGetTestDir(); private String sourcePath = baseTestDir + "/source"; private String feedDataLocation = baseTestDir + "/source" + MINUTE_DATE_PATTERN; private String targetPath = baseTestDir + "/target"; @@ -83,14 +83,14 @@ public class FeedReplicationTest extends BaseTestClass { bundles[1] = new Bundle(bundle, cluster2); bundles[2] = new Bundle(bundle, cluster3); - bundles[0].generateUniqueBundle(); - bundles[1].generateUniqueBundle(); - bundles[2].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); + bundles[1].generateUniqueBundle(this); + bundles[2].generateUniqueBundle(this); } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java index 49e4b62..ec117d7 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java @@ -49,7 +49,7 @@ public class FeedResumeTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0] = new Bundle(bundles[0], cluster); bundles[0].submitClusters(prism); feed = bundles[0].getInputFeedFromBundle(); @@ -57,7 +57,7 @@ public class FeedResumeTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java index c498807..79b722a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java @@ -46,14 +46,14 @@ public class FeedScheduleTest extends BaseTestClass { public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); Bundle.submitCluster(bundles[0]); feed = bundles[0].getInputFeedFromBundle(); } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java index 57a6164..d5e8696 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java @@ -51,7 +51,7 @@ public class FeedStatusTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0] = new Bundle(bundles[0], cluster); //submit the cluster @@ -63,7 +63,7 @@ public class FeedStatusTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java index 14f0517..dcc5a53 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java @@ -53,13 +53,13 @@ public class FeedSubmitAndScheduleTest extends BaseTestClass { public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); feed = bundles[0].getDataSets().get(0); } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(groups = {"singleCluster"}) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java index bcee706..374c442 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java @@ -42,7 +42,7 @@ public class FeedSubmitTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0] = new Bundle(bundles[0], cluster); //submit the cluster @@ -54,7 +54,7 @@ public class FeedSubmitTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java index 6b74b37..5217818 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java @@ -45,7 +45,7 @@ public class FeedSuspendTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0] = new Bundle(bundles[0], cluster); //submit the cluster @@ -58,7 +58,7 @@ public class FeedSuspendTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java index a83a319..c8b76e9 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java @@ -55,7 +55,7 @@ public class InstanceParamTest extends BaseTestClass { * test cases for https://issues.apache.org/jira/browse/FALCON-263. */ - private String baseTestHDFSDir = baseHDFSDir + "/InstanceParamTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String feedInputPath = baseTestHDFSDir + "/testInputData" + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private String startTime; @@ -77,12 +77,12 @@ public class InstanceParamTest extends BaseTestClass { public void setup() throws Exception { processBundle = BundleUtil.readELBundle(); processBundle = new Bundle(processBundle, cluster1); - processBundle.generateUniqueBundle(); + processBundle.generateUniqueBundle(this); processBundle.setInputFeedDataPath(feedInputPath); processBundle.setProcessWorkflow(aggregateWorkflowDir); for (int i = 0; i < 3; i++) { bundles[i] = new Bundle(processBundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } processName = processBundle.getProcessName(); @@ -153,7 +153,7 @@ public class InstanceParamTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() throws IOException { - removeBundles(processBundle); + removeTestClassEntities(); for (FileSystem fs : serverFS) { HadoopUtil.deleteDirIfExists(Util.getPathPrefix(feedInputPath), fs); } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java index 23f93ac..7d8cb00 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java @@ -57,7 +57,7 @@ import java.util.List; @Test(groups = "embedded") public class InstanceSummaryTest extends BaseTestClass { - private String baseTestHDFSDir = baseHDFSDir + "/InstanceSummaryTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String feedInputPath = baseTestHDFSDir + "/testInputData" + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private String startTime; @@ -84,14 +84,14 @@ public class InstanceSummaryTest extends BaseTestClass { public void setup() throws Exception { processBundle = BundleUtil.readELBundle(); processBundle = new Bundle(processBundle, cluster3); - processBundle.generateUniqueBundle(); + processBundle.generateUniqueBundle(this); processBundle.setInputFeedDataPath(feedInputPath); processBundle.setOutputFeedLocationData(baseTestHDFSDir + "/output" + MINUTE_DATE_PATTERN); processBundle.setProcessWorkflow(aggregateWorkflowDir); for (int i = 0; i < 3; i++) { bundles[i] = new Bundle(processBundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } processName = Util.readEntityName(processBundle.getProcessData()); @@ -263,6 +263,6 @@ public class InstanceSummaryTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() throws IOException { - removeBundles(processBundle); + removeTestClassEntities(); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java index b5aa514..a666796 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java @@ -73,7 +73,7 @@ public class NewRetryTest extends BaseTestClass { private OozieClient clusterOC = serverOC.get(0); private DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy/MM/dd/HH/mm"); - private final String baseTestDir = baseHDFSDir + "/NewRetryTest"; + private final String baseTestDir = cleanAndGetTestDir(); private final String aggregateWorkflowDir = baseTestDir + "/aggregator"; private final String lateDir = baseTestDir + "/lateDataTest/testFolders"; private final String latePath = lateDir + MINUTE_DATE_PATTERN; @@ -88,7 +88,7 @@ public class NewRetryTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setUp() throws Exception { bundles[0] = new Bundle(BundleUtil.readRetryBundle(), cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setProcessWorkflow(aggregateWorkflowDir); startDate = new DateTime(DateTimeZone.UTC).plusMinutes(1); endDate = new DateTime(DateTimeZone.UTC).plusMinutes(2); @@ -105,7 +105,7 @@ public class NewRetryTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(dataProvider = "DP", groups = {"0.2.2", "retry"}, enabled = true) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java index 9f23f06..59a701d 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java @@ -53,9 +53,9 @@ public class NoOutputProcessTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String testDir = baseHDFSDir + "/NoOutputProcessTest"; + private String testDir = cleanAndGetTestDir(); private String inputPath = testDir + "/input" + MINUTE_DATE_PATTERN; - private String workflowForNoIpOp = baseHDFSDir + "/PrismProcessScheduleTest/noInOp"; + private String workflowForNoIpOp = cleanAndGetTestDir(); private static final Logger LOGGER = Logger.getLogger(NoOutputProcessTest.class); @BeforeClass(alwaysRun = true) @@ -63,7 +63,7 @@ public class NoOutputProcessTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); uploadDirToClusters(workflowForNoIpOp, OSUtil.RESOURCES + "workflows/aggregatorNoOutput/"); Bundle b = BundleUtil.readELBundle(); - b.generateUniqueBundle(); + b.generateUniqueBundle(this); b = new Bundle(b, cluster); String startDate = "2010-01-03T00:00Z"; String endDate = "2010-01-03T03:00Z"; @@ -77,7 +77,7 @@ public class NoOutputProcessTest extends BaseTestClass { @BeforeMethod(alwaysRun = true) public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0] = new Bundle(bundles[0], cluster); bundles[0].setProcessWorkflow(workflowForNoIpOp); bundles[0].setInputFeedDataPath(inputPath); @@ -92,7 +92,7 @@ public class NoOutputProcessTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java index 3ce68c4..d0c9a3c 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java @@ -52,7 +52,7 @@ public class ProcessFrequencyTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String baseTestHDFSDir = baseHDFSDir + "/ProcessFrequencyTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; @BeforeClass(alwaysRun = true) @@ -60,7 +60,7 @@ public class ProcessFrequencyTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle bundle = BundleUtil.readELBundle(); - bundle.generateUniqueBundle(); + bundle.generateUniqueBundle(this); bundles[0] = new Bundle(bundle, cluster); } @@ -68,13 +68,13 @@ public class ProcessFrequencyTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setProcessWorkflow(aggregateWorkflowDir); } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java index 21dc8de..2fc4779 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java @@ -51,7 +51,7 @@ import java.util.List; @Test(groups = "embedded") public class ProcessInstanceColoMixedTest extends BaseTestClass { - private final String baseTestHDFSDir = baseHDFSDir + "/ProcessInstanceColoMixedTest"; + private final String baseTestHDFSDir = cleanAndGetTestDir(); private final String feedPath = baseTestHDFSDir + "/feed0%d" + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private ColoHelper cluster1 = servers.get(0); @@ -72,9 +72,9 @@ public class ProcessInstanceColoMixedTest extends BaseTestClass { //get 3 unique bundles bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[1] = BundleUtil.readELBundle(); - bundles[1].generateUniqueBundle(); + bundles[1].generateUniqueBundle(this); //generate bundles according to config files bundles[0] = new Bundle(bundles[0], cluster1); @@ -94,7 +94,7 @@ public class ProcessInstanceColoMixedTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } @Test(timeOut = 12000000) http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java index a45911f..231a9ca 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java @@ -55,8 +55,7 @@ public class ProcessInstanceKillsTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private OozieClient clusterOC = serverOC.get(0); private FileSystem clusterFS = serverFS.get(0); - private String testDir = "/ProcessInstanceKillsTest"; - private String baseTestHDFSDir = baseHDFSDir + testDir; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; @@ -68,7 +67,7 @@ public class ProcessInstanceKillsTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle b = BundleUtil.readELBundle(); - b.generateUniqueBundle(); + b.generateUniqueBundle(this); b = new Bundle(b, cluster); b.setInputFeedDataPath(feedInputPath); } @@ -77,7 +76,7 @@ public class ProcessInstanceKillsTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setProcessWorkflow(aggregateWorkflowDir); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setOutputFeedPeriodicity(5, TimeUnit.minutes); @@ -88,7 +87,7 @@ public class ProcessInstanceKillsTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java index bab17c2..1b1b6ca 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java @@ -50,7 +50,7 @@ import java.util.List; @Test(groups = "embedded") public class ProcessInstanceRerunTest extends BaseTestClass { - private String baseTestDir = baseHDFSDir + "/ProcessInstanceRerunTest"; + private String baseTestDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestDir + "/aggregator"; private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestDir + "/output-data" + MINUTE_DATE_PATTERN; @@ -76,7 +76,7 @@ public class ProcessInstanceRerunTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setProcessWorkflow(aggregateWorkflowDir); bundles[0].setProcessPeriodicity(5, TimeUnit.minutes); @@ -86,7 +86,7 @@ public class ProcessInstanceRerunTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java index f42ef1c..ecbc08e 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java @@ -50,7 +50,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String baseTestHDFSDir = baseHDFSDir + "/ProcessInstanceResumeTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; @@ -71,7 +71,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setOutputFeedLocationData(feedOutputPath); bundles[0].setProcessWorkflow(aggregateWorkflowDir); @@ -84,7 +84,7 @@ public class ProcessInstanceResumeTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java index d161a82..c9334eb 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java @@ -54,7 +54,7 @@ public class ProcessInstanceRunningTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); private OozieClient clusterOC = serverOC.get(0); - private String baseTestHDFSDir = baseHDFSDir + "/ProcessInstanceRunningTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; @@ -67,7 +67,7 @@ public class ProcessInstanceRunningTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle bundle = BundleUtil.readELBundle(); - bundle.generateUniqueBundle(); + bundle.generateUniqueBundle(this); bundle = new Bundle(bundle, cluster); bundle.setInputFeedDataPath(feedInputPath); } @@ -76,7 +76,7 @@ public class ProcessInstanceRunningTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setProcessWorkflow(aggregateWorkflowDir); bundles[0].setProcessValidity("2010-01-02T01:00Z", "2010-01-02T01:11Z"); @@ -88,7 +88,7 @@ public class ProcessInstanceRunningTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java index adbfd2c..e1e7dc1 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java @@ -61,7 +61,7 @@ public class ProcessInstanceStatusTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); - private String baseTestHDFSDir = baseHDFSDir + "/ProcessInstanceStatusTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; @@ -79,7 +79,7 @@ public class ProcessInstanceStatusTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); Bundle bundle = BundleUtil.readELBundle(); - bundle.generateUniqueBundle(); + bundle.generateUniqueBundle(this); bundle = new Bundle(bundle, cluster); bundle.setInputFeedDataPath(feedInputPath); } @@ -91,7 +91,7 @@ public class ProcessInstanceStatusTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setProcessWorkflow(aggregateWorkflowDir); bundles[0].setProcessValidity("2010-01-02T01:00Z", "2010-01-02T01:22Z"); @@ -101,7 +101,7 @@ public class ProcessInstanceStatusTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java index 6058713..e159d78 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java @@ -48,7 +48,7 @@ import java.io.IOException; @Test(groups = "embedded") public class ProcessInstanceSuspendTest extends BaseTestClass { - private String baseTestHDFSDir = baseHDFSDir + "/ProcessInstanceSuspendTest"; + private String baseTestHDFSDir = cleanAndGetTestDir(); private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; @@ -62,7 +62,7 @@ public class ProcessInstanceSuspendTest extends BaseTestClass { bundles[0] = BundleUtil.readELBundle(); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); + bundles[0].generateUniqueBundle(this); bundles[0].setInputFeedDataPath(feedInputPath); bundles[0].setProcessWorkflow(aggregateWorkflowDir); bundles[0].setProcessPeriodicity(5, TimeUnit.minutes); @@ -72,7 +72,7 @@ public class ProcessInstanceSuspendTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() throws IOException { - removeBundles(); + removeTestClassEntities(); HadoopUtil.deleteDirIfExists(baseTestHDFSDir, clusterFS); } http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java index 20129cb..338d90a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java @@ -46,7 +46,7 @@ public class ProcessLateRerunTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); private OozieClient cluster1OC = serverOC.get(0); private FileSystem cluster1FS = serverFS.get(0); - private String aggregateWorkflowDir = baseHDFSDir + "/ProcessLateRerunTest/aggregator"; + private String aggregateWorkflowDir = cleanAndGetTestDir() + "/aggregator"; private static final Logger LOGGER = Logger.getLogger(ProcessLateRerunTest.class); @BeforeClass(alwaysRun = true) @@ -59,14 +59,14 @@ public class ProcessLateRerunTest extends BaseTestClass { Bundle bundle = BundleUtil.readLateDataBundle(); for (int i = 0; i < 1; i++) { bundles[i] = new Bundle(bundle, servers.get(i)); - bundles[i].generateUniqueBundle(); + bundles[i].generateUniqueBundle(this); bundles[i].setProcessWorkflow(aggregateWorkflowDir); } } @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/0692b241/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java index 2dd9241..7cb98cf 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java @@ -48,7 +48,7 @@ public class ProcessLibPathTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private FileSystem clusterFS = serverFS.get(0); - private String testDir = baseHDFSDir + "/ProcessLibPath"; + private String testDir = cleanAndGetTestDir(); private String testLibDir = testDir + "/TestLib"; private static final Logger LOGGER = Logger.getLogger(ProcessLibPathTest.class); private String processName; @@ -58,7 +58,7 @@ public class ProcessLibPathTest extends BaseTestClass { public void createTestData() throws Exception { LOGGER.info("in @BeforeClass"); Bundle b = BundleUtil.readELBundle(); - b.generateUniqueBundle(); + b.generateUniqueBundle(this); b = new Bundle(b, cluster); String startDate = "2010-01-01T22:00Z"; String endDate = "2010-01-02T03:00Z"; @@ -73,12 +73,12 @@ public class ProcessLibPathTest extends BaseTestClass { public void setup() throws Exception { bundles[0] = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundles[0], cluster); - bundles[0].generateUniqueBundle(); - bundles[0].setInputFeedDataPath(baseHDFSDir + MINUTE_DATE_PATTERN); + bundles[0].generateUniqueBundle(this); + bundles[0].setInputFeedDataPath(testDir + MINUTE_DATE_PATTERN); bundles[0].setProcessValidity("2010-01-02T01:00Z", "2010-01-02T01:04Z"); bundles[0].setProcessPeriodicity(5, TimeUnit.minutes); bundles[0].setOutputFeedPeriodicity(5, TimeUnit.minutes); - bundles[0].setOutputFeedLocationData(baseHDFSDir + "/output-data" + MINUTE_DATE_PATTERN); + bundles[0].setOutputFeedLocationData(testDir + "/output-data" + MINUTE_DATE_PATTERN); bundles[0].setProcessConcurrency(1); bundles[0].setProcessLibPath(testLibDir); process = bundles[0].getProcessData(); @@ -87,7 +87,7 @@ public class ProcessLibPathTest extends BaseTestClass { @AfterMethod(alwaysRun = true) public void tearDown() { - removeBundles(); + removeTestClassEntities(); } /**
