Repository: bigtop Updated Branches: refs/heads/master 54d45491f -> b91629c91
BIGTOP-1209. Add SLive test into longevity suite Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b91629c9 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b91629c9 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b91629c9 Branch: refs/heads/master Commit: b91629c91533ab2bf74ac4448b9cb815ffa78169 Parents: 54d4549 Author: Virginia Wang <[email protected]> Authored: Thu Feb 20 17:36:06 2014 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Thu Feb 20 17:36:06 2014 -0800 ---------------------------------------------------------------------- .../bigtop/itest/iolongevity/TestSLive.groovy | 122 +++++++++++++++++++ bigtop-tests/test-execution/README | 7 +- 2 files changed, 128 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/b91629c9/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestSLive.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestSLive.groovy b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestSLive.groovy new file mode 100644 index 0000000..76a96e6 --- /dev/null +++ b/bigtop-tests/test-artifacts/longevity/src/main/groovy/org/apache/bigtop/itest/iolongevity/TestSLive.groovy @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.iolongevity + +import org.apache.bigtop.itest.JarContent +import org.apache.bigtop.itest.shell.Shell +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import static org.apache.bigtop.itest.LogErrorsUtils.logError +import static org.junit.Assert.assertNotNull +import static org.junit.Assert.assertTrue + +/** SLive test includes the following test sequence + * 100% creates, 100% renames, 100% read, 100% append, 100% ls, and 100% delete + * A mix 20% creates, 20% read, 20% append, 20% ls, 20% mkdir + */ + +public class TestSLive { + static Shell sh = new Shell("/bin/bash -s") + private static final String hadoopMapReduceHome = + System.getProperty('HADOOP_MAPRED_HOME', '/usr/lib/hadoop-mapreduce') + private static final String Slive_jar = + JarContent.getJarName(hadoopMapReduceHome, + 'hadoop.mapreduce.client.jobclient.*.tests.jar') + private static final String SLIVE_JAR = + hadoopMapReduceHome + "/" + Slive_jar + private static final int SLEEP_TIMEOUT = 5000 + private static final String SLIVE_OUTPUT_FILE = "/test/slive/slive/output" + private static final String SLIVE_ROOT_FILE = "/test/slive" + private final int numOfIterations = Integer.getInteger("numOfIterations", 1); + static String[] sliveCmds + + @BeforeClass + static void setUp() throws IOException { + assertNotNull("Can't find hadoop.mapreduce.client.jobclient.tests.jar", + Slive_jar) + final String numSliveFiles = System.getProperty("numSliveFiles", "100") + final String writeSize = System.getProperty("writeSize", "20480,20480") + final String readSize = System.getProperty("readSize", "20480,20480") + final String appendSize = System.getProperty("appendSize", "20480,20480") + final String blockSize = System.getProperty("blockSize", "10240,10240") + + String SLIVE_TEMPLATE = "hadoop jar %s SliveTest -create %s -delete %s " + + "-rename %s -read %s -append %s -ls %s -mkdir %s -files %s " + + "-writeSize %s -readSize %s -appendSize %s -blockSize %s -resFile %s" + sliveCmds = [ + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 100, 0, 0, 0, 0, 0, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputcreate.txt"), //create + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 0, 0, 100, 0, 0, 0, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputrename.txt"), //rename + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 0, 0, 0, 100, 0, 0, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputread.txt"), //read + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 0, 0, 0, 0, 100, 0, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputappend.txt"), //append + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 0, 0, 0, 0, 0, 100, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputls.txt"), //ls + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 0, 100, 0, 0, 0, 0, 0, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputdelete.txt"), //delete + String.format(SLIVE_TEMPLATE, SLIVE_JAR, 20, 0, 0, 20, 20, 20, 20, + numSliveFiles, writeSize, readSize, appendSize, blockSize, + "sliveOutputmix.txt") //mix + ] + } + + @Before + public void tearDown() { + sh.exec("hadoop fs -rm -r " + SLIVE_ROOT_FILE) + Thread.sleep(SLEEP_TIMEOUT) + } + + public void cleanup() { + sh.exec("hadoop fs -rm -r " + SLIVE_OUTPUT_FILE) + Thread.sleep(SLEEP_TIMEOUT) + } + + @Test + public void testSlive() { + String suffix = "" + for (int counter = 0; counter < numOfIterations; counter++) { + sh.exec("hadoop fs -mkdir " + SLIVE_ROOT_FILE) + sliveCmds.each { cmd -> + sh.exec(cmd + suffix) + logError(sh) + assertTrue(sh.getRet() == 0) + String expectedFile = SLIVE_OUTPUT_FILE + "/_SUCCESS" + sh.exec("hadoop fs -ls " + expectedFile) + boolean success = false + sh.getOut().each { str -> + if (str.contains(expectedFile)) { + success = true + } + } + assertTrue("File " + expectedFile + " was not found", success) + cleanup() + } + suffix = "." + counter + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/b91629c9/bigtop-tests/test-execution/README ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/README b/bigtop-tests/test-execution/README index 136cbbf..3673e3c 100644 --- a/bigtop-tests/test-execution/README +++ b/bigtop-tests/test-execution/README @@ -19,4 +19,9 @@ of Hadoop stack validations using iTest framework To run the DFSIO longevity test, you can parametrize the number of files (-nrFiles), size of files (-fileSize), and number of cycles using system properties numFiles, sizeFile, and numOfIterations, respectively. -User must make sure that the user home directory exists prior to running the test. \ No newline at end of file +User must make sure that the user home directory exists prior to running the test. + +SLive longevity test: +User must create /test directory in hdfs with the necessary permissions +Default parameters: blockSize 10240,10240 appendSize 20480,20480 writeSize 20480,20480 readSize 20480,20480 files 100 numOfIterations 0 +System parameter names: blockSize, appendSize, writeSize, readSize, numSliveFiles, numOfIterations
