Updated Branches: refs/heads/master ea1779258 -> 0f1525d14
BIGTOP-625. Add HDFS Append Test (Sujay Rau via rvs) Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4313c623 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4313c623 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4313c623 Branch: refs/heads/master Commit: 4313c6236a8a8551681a0242f59aa28fe5c13263 Parents: ea17792 Author: Roman Shaposhnik <[email protected]> Authored: Thu Apr 25 12:18:52 2013 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Thu Apr 25 12:18:52 2013 -0700 ---------------------------------------------------------------------- .../bigtop/itest/hadoop/hdfs/TestFileAppend.groovy | 253 +++++++++++++++ 1 files changed, 253 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4313c623/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestFileAppend.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestFileAppend.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestFileAppend.groovy new file mode 100644 index 0000000..8b58685 --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestFileAppend.groovy @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.itest.hadoop.hdfs; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.apache.bigtop.itest.shell.Shell; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.io.*; +import org.apache.hadoop.conf.Configuration; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestFileAppend { + + private static Shell sh = new Shell("/bin/bash -s"); + private static Shell shHDFS = new Shell("/bin/bash", "hdfs"); + private static final String HADOOP_HOME = System.getenv('HADOOP_HOME'); + private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR'); + private static final String USERNAME = System.getProperty("user.name"); + private static String date = sh.exec("date").getOut().get(0).replaceAll("\\s","").replaceAll(":",""); + private static String testAppendInput = "testAppendInput$date"; + private static String testAppendOutput = "testAppendOutput$date"; + private static String namenode; + private static Configuration conf; + + @BeforeClass + public static void setUp() { + conf = new Configuration(); + namenode = conf.get("fs.defaultFS"); + if (namenode == null) { + namenode = conf.get("fs.default.name"); + } + assertTrue("Could not find namenode", namenode != null); + + // creating test directory and test files + sh.exec("hadoop fs -mkdir $testAppendInput"); + sh.exec("echo \"-----TEST INPUT1-----\" > appendinput1.txt$date"); + sh.exec("echo \"-----TEST INPUT2-----\" > appendinput2.txt$date"); + sh.exec("echo \"-----TEST INPUT1-----\" > appendCorrect.txt$date"); + sh.exec("echo \"-----TEST INPUT2-----\" >> appendCorrect.txt$date"); + sh.exec("hadoop fs -put append* $testAppendInput"); + + System.out.println("Running File Append Test:"); + + } + + @AfterClass + public static void tearDown() { + // deletion of test folder + sh.exec("hadoop fs -test -e $testAppendInput"); + if (sh.getRet() == 0) { + sh.exec("hadoop fs -rmr -skipTrash $testAppendInput"); + assertTrue("Deletion of previous testAppendInputs from HDFS failed", + sh.getRet() == 0); + } + + } + + @Test + public void testAppendOnPreExistingFile() { + FileSystem fs = FileSystem.get(conf); + + // setting paths for I/O stream creation + String myInputPath = namenode + "/user/$USERNAME/$testAppendInput/appendinput2.txt$date"; + Path inFile = new Path(myInputPath); + assertTrue("Input file not found", fs.exists(inFile)); + String myOutputPath = namenode + "/user/$USERNAME/$testAppendInput/appendinput1.txt$date"; + Path outFile = new Path(myOutputPath); + assertTrue("Output file not found", fs.exists(outFile)); + + FSDataInputStream input1 = fs.open(inFile); + FSDataOutputStream output1 = fs.append(outFile); + + // append + IOUtils.copyBytes(input1, output1, 4096, true); + + sh.exec("hadoop fs -cat $testAppendInput/appendinput1.txt$date > $testAppendOutput"); + sh.exec("if diff $testAppendOutput appendCorrect.txt$date >/dev/null; then echo \"success\"; else echo \"failure\"; fi"); + assertTrue("Append did not work", sh.getOut().get(0).equals("success")); + sh.exec("rm -rf appendinput1.txt$date", "rm -rf appendinput2.txt$date"); + + } + + @Test + public void testAppendOnCreatedFile() { + FileSystem fs = FileSystem.get(conf); + + // setting paths for I/O stream creation + String myOutputCreate = namenode + "/user/$USERNAME/$testAppendInput/appendinput3.txt$date"; + Path outCreate = new Path(myOutputCreate); + FSDataOutputStream outputTemp = fs.create(outCreate); + String myString = "-----TEST INPUT1-----\n"; + InputStream is = new ByteArrayInputStream(myString.getBytes()); + IOUtils.copyBytes(is, outputTemp, 4096, true); + + String myInputPath = namenode + "/user/$USERNAME/$testAppendInput/appendinput2.txt$date"; + Path inFile = new Path(myInputPath); + assertTrue("Input file not found", fs.exists(inFile)); + String myOutputPath = namenode + "/user/$USERNAME/$testAppendInput/appendinput3.txt$date"; + Path outFile = new Path(myOutputPath); + assertTrue("Output file not found", fs.exists(outFile)); + + FSDataInputStream input1 = fs.open(inFile); + FSDataOutputStream output1 = fs.append(outFile); + + //append + IOUtils.copyBytes(input1, output1, 4096, true); + + sh.exec("hadoop fs -cat $testAppendInput/appendinput3.txt$date > $testAppendOutput"); + sh.exec("if diff $testAppendOutput appendCorrect.txt$date >/dev/null; then echo \"success\"; else echo \"failure\"; fi"); + assertTrue("Append did not work", sh.getOut().get(0).equals("success")); + sh.exec("rm -rf $testAppendOutput", "rm -rf appendinput1.txt$date", "rm -rf appendinput2.txt$date"); + sh.exec("rm -rf appendCorrect.txt$date"); + sh.exec("rm -rf appendinput3.txt$date"); + } + + + @Test + public void testAppendFilesGreaterThanBlockSize() { + FileSystem fs = FileSystem.get(conf); + + // creating test files that exceed block size; putting them on hdfs + sh.exec("dd if=/dev/urandom of=3mboutput.file$date count=3 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("dd if=/dev/urandom of=3mbinput.file$date count=3 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("hadoop fs -Ddfs.block.size=2097152 -put 3mb* $testAppendInput"); + assertTrue("Could not put test files onto hdfs", sh.getRet() == 0); + sh.exec("cat 3mbinput.file$date >> 3mboutput.file$date"); + + // setting paths for I/O stream creation + String myInputPath = namenode + "/user/$USERNAME/$testAppendInput/3mbinput.file$date"; + Path inFile = new Path(myInputPath); + assertTrue("Input file not found", fs.exists(inFile)); + String myOutputPath = namenode + "/user/$USERNAME/$testAppendInput/3mboutput.file$date"; + Path outFile = new Path(myOutputPath); + assertTrue("Output file not found", fs.exists(outFile)); + + FSDataInputStream input1 = fs.open(inFile); + FSDataOutputStream output1 = fs.append(outFile); + + // append + IOUtils.copyBytes(input1, output1, 4096, true); + + sh.exec("hadoop fs -cat $testAppendInput/3mboutput.file$date > $testAppendOutput"); + sh.exec("if diff $testAppendOutput 3mboutput.file$date >/dev/null; then echo \"success\"; else echo \"failure\"; fi"); + assertTrue("Append result is not what is expected", sh.getOut().get(0).equals("success")); + sh.exec("rm -rf $testAppendOutput", "rm -rf 3mboutput.file$date", "rm -rf 3mbinput.file$date"); + } + + @Test + public void testFsckSanity() { + FileSystem fs = FileSystem.get(conf); + + // test file creation + sh.exec("dd if=/dev/zero of=test1.file$date count=1 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("dd if=/dev/zero of=test2.file$date count=1 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("hadoop fs -put test1.file$date $testAppendInput", "hadoop fs -put test2.file$date $testAppendInput"); + assertTrue("Could not put test files onto hdfs", sh.getRet() == 0); + + // setting paths for I/O stream creation + String myInputPath = namenode + "/user/$USERNAME/$testAppendInput/test1.file$date"; + Path inFile = new Path(myInputPath); + assertTrue("Input file not found", fs.exists(inFile)); + String myOutputPath = namenode + "/user/$USERNAME/$testAppendInput/test2.file$date"; + Path outFile = new Path(myOutputPath); + assertTrue("Output file not found", fs.exists(outFile)); + + FSDataInputStream input1 = fs.open(inFile); + FSDataOutputStream output1 = fs.append(outFile); + + // append + IOUtils.copyBytes(input1, output1, 4096, true); + + // running fsck + shHDFS.exec("hadoop fsck /user/$USERNAME/$testAppendInput/test2.file$date"); + Boolean success = shHDFS.getOut().get(shHDFS.getOut().size() - 1).contains("is HEALTHY");; + assertTrue("Append made file unhealthy", success == true); + + sh.exec("rm -rf test1.file$date", "rm -rf test2.file$date"); + } + + @Test + public void testMultipleOutputStreamFailure() { + FileSystem fs = FileSystem.get(conf); + + // test file creation + sh.exec("dd if=/dev/zero of=test3.file$date count=1 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("dd if=/dev/zero of=test4.file$date count=1 bs=1048576"); + assertTrue("File creation error", sh.getRet() == 0); + sh.exec("hadoop fs -put test3.file$date $testAppendInput", "hadoop fs -put test4.file$date $testAppendInput"); + assertTrue("Could not put test files onto hdfs", sh.getRet() == 0); + + // setting paths for I/O stream creation + String myInputPath = namenode + "/user/$USERNAME/$testAppendInput/test3.file$date"; + Path inFile = new Path(myInputPath); + assertTrue("Input file not found", fs.exists(inFile)); + String myOutputPath = namenode + "/user/$USERNAME/$testAppendInput/test4.file$date"; + Path outFile = new Path(myOutputPath); + assertTrue("Output file not found", fs.exists(outFile)); + + FSDataInputStream input1 = fs.open(inFile); + FSDataOutputStream output1 = fs.append(outFile); + + // append + IOUtils.copyBytes(input1, output1, 4096, false); + + // attempting second output stream + try { + FSDataOutputStream output2 = fs.append(outFile); + assertTrue("Should not have been able to open second output stream", false); + IOUtils.closeStream(output2); + } + catch (Exception e) { + } + + // attempting second output stream after first stream is closed + IOUtils.closeStream(output1); + FSDataOutputStream output3 = fs.append(outFile); + + IOUtils.closeStream(output3); + IOUtils.closeStream(input1); + sh.exec("rm -rf test3.file$date", "rm -rf test4.file$date"); + assertTrue("Could not remove test files", sh.getRet() == 0); + + } + +} + +
