Repository: bigtop Updated Branches: refs/heads/master 158e74608 -> 54d45491f
BIGTOP-1208. Add DFSIO test into longevity suite Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/54d45491 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/54d45491 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/54d45491 Branch: refs/heads/master Commit: 54d45491fb56ae29df0fee8507cb96582727f70c Parents: 158e746 Author: Dasha Boudnik <[email protected]> Authored: Wed Feb 19 19:21:12 2014 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Wed Feb 19 19:21:12 2014 -0800 ---------------------------------------------------------------------- .../bigtop/itest/iolongevity/TestDFSIO.groovy | 97 ++++++++++++++++++++ bigtop-tests/test-execution/README | 4 + 2 files changed, 101 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy new file mode 100644 index 0000000..e32d56e --- /dev/null +++ b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestDFSIO.groovy @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.iolongevity + +import org.apache.bigtop.itest.JarContent +import org.apache.bigtop.itest.shell.Shell +import static org.apache.bigtop.itest.LogErrorsUtils.logError +import static org.junit.Assert.assertNotNull +import static org.junit.Assert.assertTrue +import org.junit.BeforeClass +import org.junit.Before +import org.junit.Test + +public class TestDFSIO { + static Shell sh = new Shell("/bin/bash -s"); + + private static final String WRITE_CMD = "-write"; + private static final String APPEND_CMD = "-append"; + private static final String READ_CMD = "-read"; + + private static final String BENCHMARKS_DIR = "/benchmarks/TestDFSIO"; + private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME'); + + private final String numFiles = System.getProperty("numFiles", "10"); + private final String sizeFile = System.getProperty("sizeFile", "10MB"); + private final int numOfIterations = Integer.getInteger("numOfIterations", 1); + private final int TIMEOUT = 5000; + + private String DFSIO_TEMPLATE; + private String result = "TestDFSIO_Results.log"; + + final String hadoopExamplesJar = + JarContent.getJarName(HADOOP_MAPRED_HOME, 'hadoop.*jobclient.*tests.*.jar'); + + final String hadoopTestJar = HADOOP_MAPRED_HOME + '/' + hadoopExamplesJar; + + @BeforeClass + static void setUp() throws IOException { + assertNotNull("HADOOP_MAPRED_HOME has to be set to run this test", HADOOP_MAPRED_HOME) + } + + @Before + public void cleanup() { + sh.exec("hadoop fs -rm -r " + BENCHMARKS_DIR); + Thread.sleep(TIMEOUT); + sh.exec("hadoop fs -mkdir -p " + BENCHMARKS_DIR); + assertTrue("Failed to create " + BENCHMARKS_DIR, sh.getRet() == 0); + } + + @Test + public void testDFSIO() { + DFSIO_TEMPLATE = "hadoop jar " + hadoopTestJar + " TestDFSIO %s "; + final String DFSIO_ARGS_TEMPLATE = "-nrFiles %s -fileSize %s -resFile %s"; + + final String writeSuccess = BENCHMARKS_DIR + "/io_write/_SUCCESS"; + final String appendSuccess = BENCHMARKS_DIR + "/io_append/_SUCCESS"; + final String readSuccess = BENCHMARKS_DIR + "/io_read/_SUCCESS"; + String argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result); + + for (int counter = 0; counter < numOfIterations; counter++) { + executeCmd(String.format(DFSIO_TEMPLATE + argStr, WRITE_CMD), writeSuccess); + executeCmd(String.format(DFSIO_TEMPLATE + argStr, APPEND_CMD), appendSuccess); + executeCmd(String.format(DFSIO_TEMPLATE + argStr, READ_CMD), readSuccess); + argStr = String.format(DFSIO_ARGS_TEMPLATE, numFiles, sizeFile, result + "." + counter); + } + } + + private void executeCmd(String cmd, String expectedFile){ + sh.exec(cmd); + logError(sh); + assertTrue("Command " + cmd + " is unsuccessful", sh.getRet() == 0); + sh.exec("hadoop fs -ls " + expectedFile); + boolean success = false; + sh.getOut().each { str -> + if (str.contains(expectedFile)) { + success = true; + } + } + assertTrue("File " + expectedFile + " was not found", success); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/54d45491/bigtop-tests/test-execution/README ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/README b/bigtop-tests/test-execution/README index c3e5bba..136cbbf 100644 --- a/bigtop-tests/test-execution/README +++ b/bigtop-tests/test-execution/README @@ -16,3 +16,7 @@ Repository contains the collection of starter test suites for different kinds of Hadoop stack validations using iTest framework +To run the DFSIO longevity test, you can parametrize the number of files (-nrFiles), +size of files (-fileSize), and number of cycles using system properties +numFiles, sizeFile, and numOfIterations, respectively. +User must make sure that the user home directory exists prior to running the test. \ No newline at end of file
