Author: cutting
Date: Tue Aug 13 23:04:55 2013
New Revision: 1513684
URL: http://svn.apache.org/r1513684
Log:
HADOOP-9740. Fix FsShell '-text' command to be able to read Avro files stored
in HDFS. Contributed by Allan Yan.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
- copied, changed from r1513673,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
Copied:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
(from r1513673,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java)
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java?p2=hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java&p1=hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java&r1=1513673&r2=1513684&rev=1513684&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
Tue Aug 13 23:04:55 2013
@@ -18,21 +18,26 @@
package org.apache.hadoop.fs.shell;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.StringWriter;
import java.lang.reflect.Method;
-import java.net.URI;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
+
/**
* This class tests the logic for displaying the binary formats supported
* by the Text command.
@@ -40,26 +45,44 @@ import org.junit.Test;
public class TestTextCommand {
private static final String TEST_ROOT_DIR =
System.getProperty("test.build.data", "build/test/data/") + "/testText";
- private static final String AVRO_FILENAME =
- new Path(TEST_ROOT_DIR, "weather.avro").toUri().getPath();
+ private static final Path AVRO_FILENAME = new Path(TEST_ROOT_DIR,
"weather.avro");
+ private static MiniDFSCluster cluster;
+ private static FileSystem fs;
+
+ @Before
+ public void setUp() throws IOException{
+ Configuration conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+ @After
+ public void tearDown() throws IOException{
+ if(fs != null){
+ fs.close();
+ }
+ if(cluster != null){
+ cluster.shutdown();
+ }
+ }
+
/**
* Tests whether binary Avro data files are displayed correctly.
*/
- @Test (timeout = 30000)
- public void testDisplayForAvroFiles() throws Exception {
- // Create a small Avro data file on the local file system.
+ @Test
+ public void testDisplayForAvroFiles() throws Exception {
+ // Create a small Avro data file on the HDFS.
createAvroFile(generateWeatherAvroBinaryData());
// Prepare and call the Text command's protected getInputStream method
// using reflection.
- Configuration conf = new Configuration();
- URI localPath = new URI(AVRO_FILENAME);
- PathData pathData = new PathData(localPath, conf);
+ Configuration conf = fs.getConf();
+ PathData pathData = new PathData(AVRO_FILENAME.toString(), conf);
Display.Text text = new Display.Text();
text.setConf(conf);
Method method = text.getClass().getDeclaredMethod(
- "getInputStream", PathData.class);
+ "getInputStream",
PathData.class);
method.setAccessible(true);
InputStream stream = (InputStream) method.invoke(text, pathData);
String output = inputStreamToString(stream);
@@ -87,12 +110,10 @@ public class TestTextCommand {
}
private void createAvroFile(byte[] contents) throws IOException {
- (new File(TEST_ROOT_DIR)).mkdir();
- File file = new File(AVRO_FILENAME);
- file.createNewFile();
- FileOutputStream stream = new FileOutputStream(file);
+ FSDataOutputStream stream = fs.create(AVRO_FILENAME);
stream.write(contents);
stream.close();
+ assertTrue(fs.exists(AVRO_FILENAME));
}
private byte[] generateWeatherAvroBinaryData() {
@@ -194,3 +215,4 @@ public class TestTextCommand {
}
}
+
