Repository: falcon Updated Branches: refs/heads/master 09841bbea -> c96bf897d
FALCON-1314 Unify handling of local files and directory contributed by Raghav Kumar Gautam Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/c96bf897 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/c96bf897 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/c96bf897 Branch: refs/heads/master Commit: c96bf897d403b77810b220694e7bdedf1fe3ccee Parents: 09841bb Author: Raghav Kumar Gautam <[email protected]> Authored: Thu Jul 16 14:39:11 2015 -0700 Committer: Raghav Kumar Gautam <[email protected]> Committed: Thu Jul 16 14:40:08 2015 -0700 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../falcon/regression/core/util/HadoopUtil.java | 9 ++-- .../falcon/regression/core/util/LogUtil.java | 10 ++--- .../falcon/regression/core/util/OSUtil.java | 45 +++++++++----------- .../falcon/regression/CombinedActionsTest.java | 7 +-- .../regression/EmbeddedPigScriptTest.java | 2 +- .../falcon/regression/FeedLateRerunTest.java | 5 ++- .../falcon/regression/FeedReplicationTest.java | 25 ++++++----- .../apache/falcon/regression/LogMoverTest.java | 4 +- .../apache/falcon/regression/NewRetryTest.java | 2 +- .../falcon/regression/NoOutputProcessTest.java | 2 +- .../regression/ProcessInstanceRerunTest.java | 2 +- .../falcon/regression/ProcessLateRerunTest.java | 6 ++- .../regression/ProcessLibPathLoadTest.java | 6 +-- .../falcon/regression/ProcessLibPathTest.java | 2 +- .../falcon/regression/TestngListener.java | 6 +-- .../falcon/regression/hcat/HCatProcessTest.java | 6 +-- .../regression/hcat/HCatReplicationTest.java | 2 +- .../falcon/regression/prism/FeedDelayTest.java | 2 +- .../prism/PrismFeedLateReplicationTest.java | 22 +++++----- .../PrismFeedReplicationPartitionExpTest.java | 8 ++-- .../regression/prism/PrismFeedUpdateTest.java | 2 +- .../prism/PrismProcessScheduleTest.java | 2 +- 23 files changed, 93 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index ebfc0cb..45be445 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -357,6 +357,8 @@ Trunk (Unreleased) FALCON-681 delete duplicate feed retention test from falcon regression (SamarthG) BUG FIXES + FALCON-1314 Unify handling of local files and directory (Raghav Kumar Gautam) + FALCON-1278 Submitted entity properties must have unique names (Raghav Kumar Gautam) FALCON-1300 Use xml instead of properties for log4j (Raghav Kumar Gautam via Ruslan Ostafiychuk) http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java index 2a5566a..afcb9c5 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java @@ -449,9 +449,9 @@ public final class HadoopUtil { LOGGER.info("folderData: " + folderPaths.toString()); createFolders(fs, folderPrefix, folderPaths); if (fileToBePut.equals("_SUCCESS")) { - copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.NORMAL_INPUT + "_SUCCESS"); + copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.concat(OSUtil.NORMAL_INPUT, "_SUCCESS")); } else { - copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.NORMAL_INPUT + "dataFile4.txt"); + copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile4.txt")); } } @@ -474,7 +474,7 @@ public final class HadoopUtil { } } createFolders(fs, folderPrefix, folderPaths); - copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.NORMAL_INPUT + "dataFile4.txt"); + copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile4.txt")); } /** @@ -498,7 +498,8 @@ public final class HadoopUtil { } createFolders(fs, folderPrefix, folderPaths); copyDataToFolders(fs, folderPrefix, folderPaths, - OSUtil.NORMAL_INPUT + "_SUCCESS", OSUtil.NORMAL_INPUT + "dataFile4.txt"); + OSUtil.concat(OSUtil.NORMAL_INPUT, "_SUCCESS"), + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile4.txt")); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/LogUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/LogUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/LogUtil.java index b03a13b..e4ce19d 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/LogUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/LogUtil.java @@ -82,7 +82,7 @@ public final class LogUtil { */ private void writeOneJobInfo(OozieClient oozieClient, String bundleJobId, String location, Collection<File> filter) { - final String fileName = OSUtil.getPath(location, bundleJobId + "-info.log"); + final String fileName = OSUtil.concat(location, bundleJobId + "-info.log"); final File file = new File(fileName); if (filter != null && filter.contains(file)) { return; @@ -156,7 +156,7 @@ public final class LogUtil { */ private void writeOneJobInfo(OozieClient oozieClient, String coordJobId, String location, Collection<File> filter) { - final String fileName = OSUtil.getPath(location, coordJobId + "-info.log"); + final String fileName = OSUtil.concat(location, coordJobId + "-info.log"); final File file = new File(fileName); if (filter != null && filter.contains(file)) { return; @@ -229,7 +229,7 @@ public final class LogUtil { */ private void writeOneJobInfo(OozieClient oozieClient, String wfJobId, String location, Collection<File> filter) { - final String fileName = OSUtil.getPath(location, wfJobId + "-info.log"); + final String fileName = OSUtil.concat(location, wfJobId + "-info.log"); final File file = new File(fileName); if (filter != null && filter.contains(file)) { return; @@ -300,7 +300,7 @@ public final class LogUtil { */ private static void writeOneJobLog(OozieClient oozieClient, String jobId, String location, Collection<File> filter) { - final String fileName = OSUtil.getPath(location, jobId + ".log"); + final String fileName = OSUtil.concat(location, jobId + ".log"); assert fileName != null; final File file = new File(fileName); if (filter != null && filter.contains(file)) { @@ -324,7 +324,7 @@ public final class LogUtil { public static void writeOozieLogs(ColoHelper coloHelper, String logLocation) { final OozieClient oozieClient = coloHelper.getFeedHelper().getOozieClient(); final String hostname = coloHelper.getClusterHelper().getQaHost(); - final String oozieLogLocation = OSUtil.getPath(logLocation + "oozie_logs", hostname); + final String oozieLogLocation = OSUtil.concat(logLocation + "oozie_logs", hostname); assert oozieLogLocation != null; final File directory = new File(oozieLogLocation); if (!directory.exists()) { http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java index d1f1c24..19f5f57 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java @@ -18,7 +18,7 @@ package org.apache.falcon.regression.core.util; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.io.FilenameUtils; /** * Util methods related to OS. @@ -32,28 +32,25 @@ public final class OSUtil { public static final String WIN_SU_BINARY = Config.getProperty("windows.su.binary", "ExecuteAs.exe"); - public static final String SEPARATOR = System.getProperty("file.separator", "/"); - public static final String RESOURCES = - String.format("src%stest%sresources%s", SEPARATOR, SEPARATOR, SEPARATOR); - public static final String RESOURCES_OOZIE = String.format(RESOURCES + "oozie%s", SEPARATOR); - public static final String OOZIE_EXAMPLE_INPUT_DATA = - String.format(RESOURCES + "OozieExampleInputData%s", SEPARATOR); - public static final String NORMAL_INPUT = - String.format(OOZIE_EXAMPLE_INPUT_DATA + "normalInput%s", SEPARATOR); - public static final String SINGLE_FILE = - String.format(OOZIE_EXAMPLE_INPUT_DATA + "SingleFile%s", SEPARATOR); - public static final String OOZIE_COMBINED_ACTIONS = - String.format(RESOURCES + "combinedWorkflow%s", SEPARATOR); - - public static final String OOZIE_LIB_FOLDER = - String.format(RESOURCES + "oozieLib%s", SEPARATOR); - public static final String MULTIPLE_ACTION_WORKFLOW = - String.format(RESOURCES + "MultipleActionWorkflow%s", SEPARATOR); - public static final String PIG_DIR = - String.format(RESOURCES + "pig%s", SEPARATOR); - - - public static String getPath(String... pathParts) { - return StringUtils.join(pathParts, OSUtil.SEPARATOR); + private static final String SEPARATOR = System.getProperty("file.separator", "/"); + + public static final String RESOURCES = concat("src", "test", "resources"); + public static final String RESOURCES_OOZIE = concat(RESOURCES, "oozie"); + public static final String OOZIE_EXAMPLE_INPUT_DATA = concat(RESOURCES, "OozieExampleInputData"); + public static final String NORMAL_INPUT = concat(OOZIE_EXAMPLE_INPUT_DATA, "normalInput"); + public static final String SINGLE_FILE = concat(OOZIE_EXAMPLE_INPUT_DATA, "SingleFile"); + public static final String OOZIE_COMBINED_ACTIONS = concat(RESOURCES, "combinedWorkflow"); + + public static final String OOZIE_LIB_FOLDER = concat(RESOURCES, "oozieLib"); + public static final String MULTIPLE_ACTION_WORKFLOW = concat(RESOURCES, "MultipleActionWorkflow"); + public static final String PIG_DIR = concat(RESOURCES, "pig"); + + + public static String concat(String path1, String path2, String... pathParts) { + String path = FilenameUtils.concat(path1, path2); + for (String pathPart : pathParts) { + path = FilenameUtils.concat(path, pathPart); + } + return path; } } http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/CombinedActionsTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/CombinedActionsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/CombinedActionsTest.java index 0c24f7e..4d432fd 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/CombinedActionsTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/CombinedActionsTest.java @@ -61,8 +61,8 @@ public class CombinedActionsTest extends BaseTestClass { private final String outputHDFSDir = baseTestHDFSDir + "/output"; private String aggregateWorkflowDir = cleanAndGetTestDir() + "/aggregator"; private static final Logger LOGGER = Logger.getLogger(CombinedActionsTest.class); - private static final String HCATDIR = OSUtil.getPath("src", "test", "resources", "hcat"); - private static final String LOCALHCATDATA = OSUtil.getPath(HCATDIR, "data"); + private static final String HCATDIR = OSUtil.concat("src", "test", "resources", "hcat"); + private static final String LOCALHCATDATA = OSUtil.concat(HCATDIR, "data"); public static final String DBNAME = "default"; public static final String COL1NAME = "id"; public static final String COL2NAME = "value"; @@ -119,7 +119,8 @@ public class CombinedActionsTest extends BaseTestClass { HadoopUtil.deleteDirIfExists(pigMrTestDir + "/input", clusterFS); List<String> dataDates = TimeUtil.getMinuteDatesOnEitherSide(startDate, endDate, 20); - HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.NORMAL_INPUT, pigMrTestDir + "/input", dataDates); + HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.concat(OSUtil.NORMAL_INPUT, pigMrTestDir, "input"), + dataDates); final String datePattern = StringUtils.join(new String[] { "yyyy", "MM", "dd", "HH", "mm"}, "-"); dataDates = TimeUtil.getMinuteDatesOnEitherSide(startDate, endDate, 60, DateTimeFormat.forPattern(datePattern)); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java index 87fab7c..c49c381 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java @@ -72,7 +72,7 @@ public class EmbeddedPigScriptTest extends BaseTestClass { LOGGER.info("in @BeforeClass"); //copy pig script - HadoopUtil.uploadDir(clusterFS, pigScriptDir, OSUtil.RESOURCES + "pig"); + HadoopUtil.uploadDir(clusterFS, pigScriptDir, OSUtil.concat(OSUtil.RESOURCES, "pig")); Bundle bundle = BundleUtil.readELBundle(); bundle.generateUniqueBundle(this); bundle = new Bundle(bundle, cluster); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java index d3370b4..59a9921 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java @@ -190,7 +190,7 @@ public class FeedLateRerunTest extends BaseTestClass { for (String location : missingDependencies) { if (tempCount==1) { LOGGER.info("Transferring data to : " + location); - HadoopUtil.copyDataToFolder(sourceFS, location, OSUtil.NORMAL_INPUT + "dataFile.xml"); + HadoopUtil.copyDataToFolder(sourceFS, location, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); tempCount++; } } @@ -203,7 +203,8 @@ public class FeedLateRerunTest extends BaseTestClass { for (String dependency : missingDependencies) { if (tempCounter==1) { LOGGER.info("Transferring late data to : " + dependency); - HadoopUtil.copyDataToFolder(sourceFS, dependency, OSUtil.NORMAL_INPUT + "dataFile.properties"); + HadoopUtil.copyDataToFolder(sourceFS, dependency, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.properties")); } tempCounter++; } http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index a7a2ea8..9ac9f24 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -144,8 +144,10 @@ public class FeedReplicationTest extends BaseTestClass { Path toSource = new Path(sourceLocation); Path toTarget = new Path(targetLocation); if (dataFlag) { - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml"); - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt"); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile1.txt")); } //check if coordinator exists @@ -232,8 +234,10 @@ public class FeedReplicationTest extends BaseTestClass { Path toTarget = new Path(targetLocation); if (dataFlag) { - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml"); - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt"); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile1.txt")); } //check if all coordinators exist @@ -329,8 +333,10 @@ public class FeedReplicationTest extends BaseTestClass { Path toSource = new Path(sourceLocation); Path toTarget = new Path(targetLocation); if (dataFlag) { - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml"); - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt"); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile1.txt")); } //check while instance is got created @@ -347,8 +353,7 @@ public class FeedReplicationTest extends BaseTestClass { LOGGER.info("Replication didn't start."); //create availability flag on source - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, - OSUtil.RESOURCES + availabilityFlagName); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.RESOURCES, availabilityFlagName)); //check if instance become running InstanceUtil.waitTillInstanceReachState(cluster2OC, feed.getName(), 1, @@ -424,8 +429,8 @@ public class FeedReplicationTest extends BaseTestClass { Path toSource = new Path(sourceLocation); Path toTarget = new Path(targetLocation); - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile.xml"); - HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.NORMAL_INPUT + "dataFile1.txt"); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile1.txt")); //check if coordinator exists InstanceUtil.waitTillInstancesAreCreated(cluster2OC, feed.toString(), 0); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java index 5398581..f936305 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java @@ -73,7 +73,7 @@ public class LogMoverTest extends BaseTestClass { LOGGER.info("startDate : " + startDate + " , endDate : " + endDate); //copy pig script and workflow - HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES + "LogMover"); + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.concat(OSUtil.RESOURCES, "LogMover")); Bundle bundle = BundleUtil.readELBundle(); bundles[0] = new Bundle(bundle, cluster); bundles[0].generateUniqueBundle(this); @@ -116,7 +116,7 @@ public class LogMoverTest extends BaseTestClass { AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, process, Job.Status.RUNNING); //Copy data to let pig job succeed - HadoopUtil.copyDataToFolder(clusterFS, propPath, OSUtil.RESOURCES + "pig"); + HadoopUtil.copyDataToFolder(clusterFS, propPath, OSUtil.concat(OSUtil.RESOURCES, "pig")); InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 0); OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java index 081938f..03bc358 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NewRetryTest.java @@ -837,7 +837,7 @@ public class NewRetryTest extends BaseTestClass { Util.findFolderBetweenGivenTimeStamps(now, now.plusMinutes(5), initialData); LOGGER.info("inserting data in folder " + insertionFolder + " at " + DateTime.now()); HadoopUtil.injectMoreData(clusterFS, lateDir + insertionFolder, - OSUtil.OOZIE_EXAMPLE_INPUT_DATA + "lateData"); + OSUtil.concat(OSUtil.OOZIE_EXAMPLE_INPUT_DATA, "lateData")); //now to validate all failed instances to check if they were retried or not. validateRetry(clusterOC, bundleId, bundles[0].getProcessObject().getRetry().getAttempts()); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java index 61c076b..0711e8a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/NoOutputProcessTest.java @@ -61,7 +61,7 @@ public class NoOutputProcessTest extends BaseTestClass { @BeforeClass(alwaysRun = true) public void createTestData() throws Exception { LOGGER.info("in @BeforeClass"); - uploadDirToClusters(workflowForNoIpOp, OSUtil.RESOURCES + "workflows/aggregatorNoOutput/"); + uploadDirToClusters(workflowForNoIpOp, OSUtil.concat(OSUtil.RESOURCES, "workflows", "aggregatorNoOutput")); Bundle b = BundleUtil.readELBundle(); b.generateUniqueBundle(this); b = new Bundle(b, cluster); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java index ef3d8a7..a8cc2e0 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java @@ -416,7 +416,7 @@ public class ProcessInstanceRerunTest extends BaseTestClass { bundles[0].setProcessData(processElement.toString()); HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.MULTIPLE_ACTION_WORKFLOW); - HadoopUtil.copyDataToFolder(clusterFS, aggregateWorkflowDir, OSUtil.PIG_DIR + "id.pig"); + HadoopUtil.copyDataToFolder(clusterFS, aggregateWorkflowDir, OSUtil.concat(OSUtil.PIG_DIR, "id.pig")); bundles[0].setProcessValidity("2015-01-02T01:00Z", "2015-01-02T01:04Z"); bundles[0].setOutputFeedLocationData(feedOutputPath); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java index aa9bdd5..554588d 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java @@ -296,7 +296,8 @@ public class ProcessLateRerunTest extends BaseTestClass { for (String location : missingDependencies) { if (tempCount==1) { LOGGER.info("Transferring data to : " + location); - HadoopUtil.copyDataToFolder(clusterFS, location, OSUtil.NORMAL_INPUT + "dataFile.xml"); + HadoopUtil.copyDataToFolder(clusterFS, location, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); tempCount++; } } @@ -314,7 +315,8 @@ public class ProcessLateRerunTest extends BaseTestClass { for (String dependency : missingDependencies) { if (tempCounter==dataFolder) { LOGGER.info("Transferring late data to : " + dependency); - HadoopUtil.copyDataToFolder(clusterFS, dependency, OSUtil.NORMAL_INPUT + "dataFile.properties"); + HadoopUtil.copyDataToFolder(clusterFS, dependency, + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.properties")); } tempCounter++; } http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java index d4da519..bd9d43a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java @@ -54,14 +54,14 @@ public class ProcessLibPathLoadTest extends BaseTestClass { private String oozieLib = MerlinConstants.OOZIE_EXAMPLE_LIB; private String oozieLibName = oozieLib.substring(oozieLib.lastIndexOf('/') + 1); - private String filename = OSUtil.OOZIE_LIB_FOLDER + "lib/" + oozieLibName; + private String filename = OSUtil.concat(OSUtil.OOZIE_LIB_FOLDER, "lib", oozieLibName); private String processName; private String process; @BeforeClass(alwaysRun = true) public void createTestData() throws Exception { LOGGER.info("in @BeforeClass"); - FileUtils.forceMkdir(new File(OSUtil.OOZIE_LIB_FOLDER + "lib")); + FileUtils.forceMkdir(new File(OSUtil.concat(OSUtil.OOZIE_LIB_FOLDER, "lib"))); saveUrlToFile(oozieLib); } @@ -91,7 +91,7 @@ public class ProcessLibPathLoadTest extends BaseTestClass { public void deleteJar() throws IOException { File file = new File(filename); Assert.assertEquals(file.delete(), true, filename + " is not present."); - FileUtils.deleteDirectory(new File(OSUtil.OOZIE_LIB_FOLDER + "lib")); + FileUtils.deleteDirectory(new File(OSUtil.concat(OSUtil.OOZIE_LIB_FOLDER, "lib"))); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java index dc2fc37..4196d99 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java @@ -120,7 +120,7 @@ public class ProcessLibPathTest extends BaseTestClass { HadoopUtil.uploadDir(clusterFS, workflowDir, OSUtil.RESOURCES_OOZIE); HadoopUtil.recreateDir(clusterFS, workflowDir + "/lib"); HadoopUtil.copyDataToFolder(clusterFS, workflowDir + "/lib/invalid.jar", - OSUtil.NORMAL_INPUT + "dataFile.xml"); + OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); bundles[0].setProcessWorkflow(workflowDir); LOGGER.info("processData: " + Util.prettyPrintXml(process)); bundles[0].submitFeedsScheduleProcess(prism); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java index 277dff5..9ea8471 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java @@ -118,13 +118,13 @@ public class TestngListener implements ITestListener, IExecutionListener { } private void takeScreenShot(ITestResult result) throws IOException { - String logs = Config.getProperty("log.capture.location", OSUtil.getPath("target", "surefire-reports")); + String logs = Config.getProperty("log.capture.location", OSUtil.concat("target", "surefire-reports")); if (BaseUITestClass.getDriver() != null) { byte[] scrFile = ((TakesScreenshot)BaseUITestClass.getDriver()).getScreenshotAs(OutputType.BYTES); String params = Arrays.toString(result.getParameters()); params = params.replaceAll("[<>\":\\\\/\\|\\?\\*]", ""); //remove <>:"/\|?* - String filename = OSUtil.getPath(logs, "screenshots", - String.format("%s.%s(%s).png", result .getTestClass().getRealClass().getSimpleName(), + String filename = OSUtil.concat(logs, "screenshots", + String.format("%s.%s(%s).png", result.getTestClass().getRealClass().getSimpleName(), result.getName(), params)); LOGGER.info("Saving screenshot to: " + filename); FileUtils.writeByteArrayToFile(new File(filename), scrFile); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java index 03e1797..0686081 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatProcessTest.java @@ -98,9 +98,9 @@ public class HCatProcessTest extends BaseTestClass { private final String col2Name = "value"; private final String partitionColumn = "dt"; - private final String hcatDir = OSUtil.getPath("src", "test", "resources", "hcat"); - private final String localHCatData = OSUtil.getPath(hcatDir, "data"); - private final String hiveScript = OSUtil.getPath(hcatDir, "hivescript"); + private final String hcatDir = OSUtil.concat("src", "test", "resources", "hcat"); + private final String localHCatData = OSUtil.concat(hcatDir, "data"); + private final String hiveScript = OSUtil.concat(hcatDir, "hivescript"); private final String startDate = "2010-01-01T20:00Z"; private final String endDate = "2010-01-01T21:10Z"; http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java index 59a2077..237b3b2 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplicationTest.java @@ -85,7 +85,7 @@ public class HCatReplicationTest extends BaseTestClass { private final String baseTestHDFSDir = cleanAndGetTestDir(); private final String dbName = "default"; - private final String localHCatData = OSUtil.getPath(OSUtil.RESOURCES, "hcat", "data"); + private final String localHCatData = OSUtil.concat(OSUtil.RESOURCES, "hcat", "data"); private static final double TIMEOUT = 15; @BeforeClass(alwaysRun = true) http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayTest.java index 22aabfa..c000561 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/FeedDelayTest.java @@ -191,7 +191,7 @@ public class FeedDelayTest extends BaseTestClass { //Adding data to empty folders for (String location : missingDependencies) { LOGGER.info("Transferring data to : " + location); - HadoopUtil.copyDataToFolder(sourceFS, location, OSUtil.RESOURCES + "dataFile.xml"); + HadoopUtil.copyDataToFolder(sourceFS, location, OSUtil.concat(OSUtil.RESOURCES, "dataFile.xml")); } return missingDependencies; } http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java index 1bc4027..4864f06 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedLateReplicationTest.java @@ -287,10 +287,10 @@ public class PrismFeedLateReplicationTest extends BaseTestClass { TimeUtil.sleepSeconds(30); //put data for the second time - HadoopUtil.flattenAndPutDataInFolder(cluster2FS, OSUtil.OOZIE_EXAMPLE_INPUT_DATA - + "2ndLateData", inputFolderListForColo1); - HadoopUtil.flattenAndPutDataInFolder(cluster3FS, OSUtil.OOZIE_EXAMPLE_INPUT_DATA - + "2ndLateData", inputFolderListForColo2); + HadoopUtil.flattenAndPutDataInFolder(cluster2FS, + OSUtil.concat(OSUtil.OOZIE_EXAMPLE_INPUT_DATA, "2ndLateData"), inputFolderListForColo1); + HadoopUtil.flattenAndPutDataInFolder(cluster3FS, + OSUtil.concat(OSUtil.OOZIE_EXAMPLE_INPUT_DATA, "2ndLateData"), inputFolderListForColo2); //sleep till late 2 starts TimeUtil.sleepTill(TimeUtil.addMinsToTime(startTime, 9)); @@ -459,10 +459,10 @@ public class PrismFeedLateReplicationTest extends BaseTestClass { } //put data for the second time - HadoopUtil.flattenAndPutDataInFolder(cluster2FS, OSUtil.OOZIE_EXAMPLE_INPUT_DATA - + "2ndLateData", inputFolderListForColo1); - HadoopUtil.flattenAndPutDataInFolder(cluster3FS, OSUtil.OOZIE_EXAMPLE_INPUT_DATA - + "2ndLateData", inputFolderListForColo2); + HadoopUtil.flattenAndPutDataInFolder(cluster2FS, OSUtil.concat(OSUtil.OOZIE_EXAMPLE_INPUT_DATA, "2ndLateData"), + inputFolderListForColo1); + HadoopUtil.flattenAndPutDataInFolder(cluster3FS, OSUtil.concat(OSUtil.OOZIE_EXAMPLE_INPUT_DATA, "2ndLateData"), + inputFolderListForColo2); //sleep till late 2 starts TimeUtil.sleepTill(TimeUtil.addMinsToTime(startTime, 9)); @@ -625,10 +625,8 @@ public class PrismFeedLateReplicationTest extends BaseTestClass { Assert.assertTrue(HadoopUtil.isFilePresentHDFS(cluster1FS, outPutLocation, "_SUCCESS")); - HadoopUtil.flattenAndPutDataInFolder(cluster2FS, OSUtil.NORMAL_INPUT, - inputFolderListForColo1); - HadoopUtil.flattenAndPutDataInFolder(cluster3FS, OSUtil.NORMAL_INPUT, - inputFolderListForColo2); + HadoopUtil.flattenAndPutDataInFolder(cluster2FS, OSUtil.NORMAL_INPUT, inputFolderListForColo1); + HadoopUtil.flattenAndPutDataInFolder(cluster3FS, OSUtil.NORMAL_INPUT, inputFolderListForColo2); //sleep till late starts TimeUtil.sleepTill(TimeUtil.addMinsToTime(startTime, 4)); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java index 43aafdf..0a97c89 100755 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationPartitionExpTest.java @@ -70,10 +70,10 @@ public class PrismFeedReplicationPartitionExpTest extends BaseTestClass { private String testDirWithDate = testBaseDir1 + testDate; private String testDirWithDateSourceTarget = testBaseDir4 + testDate; private String testDirWithDateSource1 = testBaseDirServer1Source + testDate; - private String testFile1 = OSUtil.NORMAL_INPUT + "dataFile.xml"; - private String testFile2 = OSUtil.RESOURCES + OSUtil.getPath("pig", "id.pig"); - private String testFile3 = OSUtil.RESOURCES + OSUtil.getPath("ELbundle", "cluster-0.1.xml"); - private String testFile4 = OSUtil.NORMAL_INPUT + "dataFile.properties"; + private String testFile1 = OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml"); + private String testFile2 = OSUtil.concat(OSUtil.RESOURCES, "pig", "id.pig"); + private String testFile3 = OSUtil.concat(OSUtil.RESOURCES, "ELbundle", "cluster-0.1.xml"); + private String testFile4 = OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.properties"); private static final Logger LOGGER = Logger.getLogger(PrismFeedReplicationPartitionExpTest.class); http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java index 5e868e2..ed48265 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java @@ -72,7 +72,7 @@ public class PrismFeedUpdateTest extends BaseTestClass { @BeforeClass(alwaysRun = true) public void uploadWorkflow() throws Exception { uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); - uploadDirToClusters(workflowForNoIpOp, OSUtil.RESOURCES + "workflows/aggregatorNoOutput/"); + uploadDirToClusters(workflowForNoIpOp, OSUtil.concat(OSUtil.RESOURCES, "workflows", "aggregatorNoOutput")); } @BeforeMethod(alwaysRun = true) http://git-wip-us.apache.org/repos/asf/falcon/blob/c96bf897/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java index 819438f..2dd602a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java @@ -62,7 +62,7 @@ public class PrismProcessScheduleTest extends BaseTestClass { @BeforeClass(alwaysRun = true) public void uploadWorkflow() throws Exception { uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); - uploadDirToClusters(workflowForNoIpOp, OSUtil.RESOURCES+"workflows/aggregatorNoOutput/"); + uploadDirToClusters(workflowForNoIpOp, OSUtil.concat(OSUtil.RESOURCES, "workflows", "aggregatorNoOutput")); } @BeforeMethod(alwaysRun = true)
