Author: cws
Date: Wed Jul 30 01:22:46 2014
New Revision: 1614534
URL: http://svn.apache.org/r1614534
Log:
HIVE-7286: Parameterize HCatMapReduceTest for testing against all Hive storage
formats (David Chen via cws)
Added:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java
Removed:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/fileformats/
Modified:
hive/trunk/.gitignore
hive/trunk/hcatalog/core/pom.xml
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
Modified: hive/trunk/.gitignore
URL:
http://svn.apache.org/viewvc/hive/trunk/.gitignore?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
--- hive/trunk/.gitignore (original)
+++ hive/trunk/.gitignore Wed Jul 30 01:22:46 2014
@@ -13,6 +13,7 @@ common/src/gen
*.iml
*.ipr
*.iws
+*.swp
derby.log
datanucleus.log
.arc
Modified: hive/trunk/hcatalog/core/pom.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/pom.xml?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/pom.xml (original)
+++ hive/trunk/hcatalog/core/pom.xml Wed Jul 30 01:22:46 2014
@@ -60,6 +60,13 @@
<artifactId>hive-exec</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<!-- inter-project -->
<dependency>
<groupId>com.google.guava</groupId>
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatMapReduceTest.java
Wed Jul 30 01:22:46 2014
@@ -19,13 +19,15 @@
package org.apache.hive.hcatalog.mapreduce;
+import com.google.common.collect.ImmutableSet;
+
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
-import junit.framework.Assert;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -40,10 +42,10 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
-import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
-import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
+import org.apache.hadoop.hive.ql.io.StorageFormats;
+import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -53,15 +55,23 @@ import org.apache.hadoop.mapreduce.JobSt
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.DefaultHCatRecord;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
+
+import junit.framework.Assert;
+
import org.junit.After;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,42 +79,62 @@ import static org.junit.Assert.assertTru
/**
* Test for HCatOutputFormat. Writes a partition using HCatOutputFormat and
reads
- * it back using HCatInputFormat, checks the column values and counts.
+ * it back using HCatInputFormat, checks the column values and counts. This
class
+ * can be tested to test different partitioning schemes.
+ *
+ * This is a parameterized test that tests HCatOutputFormat and
HCatInputFormat against Hive's
+ * native storage formats enumerated using {@link
org.apache.hive.hcatalog.mapreduce.StorageFormats}.
*/
+@RunWith(Parameterized.class)
public abstract class HCatMapReduceTest extends HCatBaseTest {
-
private static final Logger LOG =
LoggerFactory.getLogger(HCatMapReduceTest.class);
+
protected static String dbName = MetaStoreUtils.DEFAULT_DATABASE_NAME;
- protected static String tableName = "testHCatMapReduceTable";
+ protected static final String TABLE_NAME = "testHCatMapReduceTable";
private static List<HCatRecord> writeRecords = new ArrayList<HCatRecord>();
private static List<HCatRecord> readRecords = new ArrayList<HCatRecord>();
- protected abstract List<FieldSchema> getPartitionKeys();
-
- protected abstract List<FieldSchema> getTableColumns();
-
private static FileSystem fs;
private String externalTableLocation = null;
+ protected String tableName;
+ protected String serdeClass;
+ protected String inputFormatClass;
+ protected String outputFormatClass;
- protected Boolean isTableExternal() {
- return false;
+ /**
+ * List of SerDe classes that the HCatalog core tests will not be run
against.
+ */
+ public static final Set<String> DISABLED_SERDES = ImmutableSet.of(
+ AvroSerDe.class.getName(),
+ ParquetHiveSerDe.class.getName());
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> generateParameters() {
+ return StorageFormats.asParameters();
}
- protected boolean isTableImmutable() {
- return true;
+ /**
+ * Test constructor that sets the storage format class names provided by the
test parameter.
+ */
+ public HCatMapReduceTest(String name, String serdeClass, String
inputFormatClass,
+ String outputFormatClass) throws Exception {
+ this.serdeClass = serdeClass;
+ this.inputFormatClass = inputFormatClass;
+ this.outputFormatClass = outputFormatClass;
+ this.tableName = TABLE_NAME + "_" + name;
}
- protected String inputFormat() {
- return RCFileInputFormat.class.getName();
- }
+ protected abstract List<FieldSchema> getPartitionKeys();
+
+ protected abstract List<FieldSchema> getTableColumns();
- protected String outputFormat() {
- return RCFileOutputFormat.class.getName();
+ protected Boolean isTableExternal() {
+ return false;
}
- protected String serdeClass() {
- return ColumnarSerDe.class.getName();
+ protected boolean isTableImmutable() {
+ return true;
}
@BeforeClass
@@ -143,13 +173,16 @@ public abstract class HCatMapReduceTest
@Before
public void createTable() throws Exception {
- String databaseName = (dbName == null) ?
MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName;
+ // Use Junit's Assume to skip running this fixture against any storage
formats whose
+ // SerDe is in the disabled serdes list.
+ Assume.assumeTrue(!DISABLED_SERDES.contains(serdeClass));
+ String databaseName = (dbName == null) ?
MetaStoreUtils.DEFAULT_DATABASE_NAME : dbName;
try {
client.dropTable(databaseName, tableName);
} catch (Exception e) {
- } //can fail with NoSuchObjectException
-
+ // Can fail with NoSuchObjectException.
+ }
Table tbl = new Table();
tbl.setDbName(databaseName);
@@ -160,10 +193,9 @@ public abstract class HCatMapReduceTest
tbl.setTableType(TableType.MANAGED_TABLE.toString());
}
StorageDescriptor sd = new StorageDescriptor();
-
sd.setCols(getTableColumns());
- tbl.setPartitionKeys(getPartitionKeys());
+ tbl.setPartitionKeys(getPartitionKeys());
tbl.setSd(sd);
sd.setBucketCols(new ArrayList<String>(2));
@@ -171,12 +203,12 @@ public abstract class HCatMapReduceTest
sd.getSerdeInfo().setName(tbl.getTableName());
sd.getSerdeInfo().setParameters(new HashMap<String, String>());
sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT,
"1");
- if (isTableExternal()){
+ if (isTableExternal()) {
sd.getSerdeInfo().getParameters().put("EXTERNAL", "TRUE");
}
- sd.getSerdeInfo().setSerializationLib(serdeClass());
- sd.setInputFormat(inputFormat());
- sd.setOutputFormat(outputFormat());
+ sd.getSerdeInfo().setSerializationLib(serdeClass);
+ sd.setInputFormat(inputFormatClass);
+ sd.setOutputFormat(outputFormatClass);
Map<String, String> tableParams = new HashMap<String, String>();
if (isTableExternal()) {
@@ -190,68 +222,59 @@ public abstract class HCatMapReduceTest
client.createTable(tbl);
}
- //Create test input file with specified number of rows
+ /*
+ * Create test input file with specified number of rows
+ */
private void createInputFile(Path path, int rowCount) throws IOException {
-
if (fs.exists(path)) {
fs.delete(path, true);
}
FSDataOutputStream os = fs.create(path);
-
for (int i = 0; i < rowCount; i++) {
os.writeChars(i + "\n");
}
-
os.close();
}
- public static class MapCreate extends
- Mapper<LongWritable, Text, BytesWritable, HCatRecord> {
-
- static int writeCount = 0; //test will be in local mode
+ public static class MapCreate extends Mapper<LongWritable, Text,
BytesWritable, HCatRecord> {
+ // Test will be in local mode.
+ static int writeCount = 0;
@Override
- public void map(LongWritable key, Text value, Context context
- ) throws IOException, InterruptedException {
- {
- try {
- HCatRecord rec = writeRecords.get(writeCount);
- context.write(null, rec);
- writeCount++;
-
- } catch (Exception e) {
-
- e.printStackTrace(System.err); //print since otherwise exception is
lost
- throw new IOException(e);
- }
+ public void map(LongWritable key, Text value, Context context)
+ throws IOException, InterruptedException {
+ try {
+ HCatRecord rec = writeRecords.get(writeCount);
+ context.write(null, rec);
+ writeCount++;
+ } catch (Exception e) {
+ // Print since otherwise exception is lost.
+ e.printStackTrace(System.err);
+ throw new IOException(e);
}
}
}
- public static class MapRead extends
- Mapper<WritableComparable, HCatRecord, BytesWritable, Text> {
-
+ public static class MapRead extends Mapper<WritableComparable, HCatRecord,
BytesWritable, Text> {
static int readCount = 0; //test will be in local mode
@Override
- public void map(WritableComparable key, HCatRecord value, Context context
- ) throws IOException, InterruptedException {
- {
- try {
- readRecords.add(value);
- readCount++;
- } catch (Exception e) {
- e.printStackTrace(); //print since otherwise exception is lost
- throw new IOException(e);
- }
+ public void map(WritableComparable key, HCatRecord value, Context context)
+ throws IOException, InterruptedException {
+ try {
+ readRecords.add(value);
+ readCount++;
+ } catch (Exception e) {
+ // Print since otherwise exception is lost.
+ e.printStackTrace();
+ throw new IOException(e);
}
}
}
- Job runMRCreate(Map<String, String> partitionValues,
- List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
- int writeCount, boolean assertWrite) throws Exception {
+ Job runMRCreate(Map<String, String> partitionValues, List<HCatFieldSchema>
partitionColumns,
+ List<HCatRecord> records, int writeCount, boolean assertWrite) throws
Exception {
return runMRCreate(partitionValues, partitionColumns, records, writeCount,
assertWrite,
true, null);
}
@@ -267,10 +290,9 @@ public abstract class HCatMapReduceTest
* @return
* @throws Exception
*/
- Job runMRCreate(Map<String, String> partitionValues,
- List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
- int writeCount, boolean assertWrite, boolean asSingleMapTask,
- String customDynamicPathPattern) throws Exception {
+ Job runMRCreate(Map<String, String> partitionValues, List<HCatFieldSchema>
partitionColumns,
+ List<HCatRecord> records, int writeCount, boolean assertWrite, boolean
asSingleMapTask,
+ String customDynamicPathPattern) throws Exception {
writeRecords = records;
MapCreate.writeCount = 0;
@@ -355,7 +377,6 @@ public abstract class HCatMapReduceTest
* @throws Exception
*/
List<HCatRecord> runMRRead(int readCount, String filter) throws Exception {
-
MapRead.readCount = 0;
readRecords.clear();
@@ -388,9 +409,7 @@ public abstract class HCatMapReduceTest
return readRecords;
}
-
protected HCatSchema getTableSchema() throws Exception {
-
Configuration conf = new Configuration();
Job job = new Job(conf, "hcat mapreduce read schema test");
job.setJarByClass(this.getClass());
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -37,8 +37,10 @@ import org.apache.hive.hcatalog.data.Def
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils;
+
import org.junit.BeforeClass;
import org.junit.Test;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,9 +55,10 @@ public class TestHCatDynamicPartitioned
protected static final int NUM_RECORDS = 20;
protected static final int NUM_PARTITIONS = 5;
- @BeforeClass
- public static void generateInputData() throws Exception {
- tableName = "testHCatDynamicPartitionedTable";
+ public TestHCatDynamicPartitioned(String formatName, String serdeClass,
String inputFormatClass,
+ String outputFormatClass) throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ tableName = "testHCatDynamicPartitionedTable_" + formatName;
generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
generateDataColumns();
}
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalDynamicPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -24,18 +24,20 @@ import org.junit.Test;
public class TestHCatExternalDynamicPartitioned extends
TestHCatDynamicPartitioned {
+ public TestHCatExternalDynamicPartitioned(String formatName, String
serdeClass,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ tableName = "testHCatExternalDynamicPartitionedTable_" + formatName;
+ generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
+ generateDataColumns();
+ }
+
@Override
protected Boolean isTableExternal() {
return true;
}
- @BeforeClass
- public static void generateInputData() throws Exception {
- tableName = "testHCatExternalDynamicPartitionedTable";
- generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
- generateDataColumns();
- }
-
/**
* Run the external dynamic partitioning test but with single map task
* @throws Exception
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalNonPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -20,6 +20,11 @@
package org.apache.hive.hcatalog.mapreduce;
public class TestHCatExternalNonPartitioned extends TestHCatNonPartitioned {
+ public TestHCatExternalNonPartitioned(String formatName, String serdeName,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeName, inputFormatClass, outputFormatClass);
+ }
@Override
protected Boolean isTableExternal() {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatExternalPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -20,6 +20,11 @@
package org.apache.hive.hcatalog.mapreduce;
public class TestHCatExternalPartitioned extends TestHCatPartitioned {
+ public TestHCatExternalPartitioned(String formatName, String serdeClass,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ }
@Override
protected Boolean isTableExternal() {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableDynamicPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -20,6 +20,11 @@
package org.apache.hive.hcatalog.mapreduce;
public class TestHCatMutableDynamicPartitioned extends
TestHCatDynamicPartitioned {
+ public TestHCatMutableDynamicPartitioned(String formatName, String
serdeClass,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ }
@Override
protected boolean isTableImmutable() {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutableNonPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -20,7 +20,11 @@
package org.apache.hive.hcatalog.mapreduce;
public class TestHCatMutableNonPartitioned extends TestHCatNonPartitioned {
-
+ public TestHCatMutableNonPartitioned(String formatName, String serdeClass,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ }
@Override
protected boolean isTableImmutable() {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMutablePartitioned.java
Wed Jul 30 01:22:46 2014
@@ -20,6 +20,11 @@
package org.apache.hive.hcatalog.mapreduce;
public class TestHCatMutablePartitioned extends TestHCatPartitioned {
+ public TestHCatMutablePartitioned(String formatName, String serdeClass,
+ String inputFormatClass, String outputFormatClass)
+ throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ }
@Override
protected boolean isTableImmutable() {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatNonPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -43,16 +43,14 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNull;
public class TestHCatNonPartitioned extends HCatMapReduceTest {
-
private static List<HCatRecord> writeRecords;
static List<HCatFieldSchema> partitionColumns;
- @BeforeClass
- public static void oneTimeSetUp() throws Exception {
-
+ public TestHCatNonPartitioned(String formatName, String serdeClass, String
inputFormatClass,
+ String outputFormatClass) throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
dbName = null; //test if null dbName works ("default" is used)
- tableName = "testHCatNonPartitionedTable";
-
+ tableName = "testHCatNonPartitionedTable_" + formatName;
writeRecords = new ArrayList<HCatRecord>();
for (int i = 0; i < 20; i++) {
Modified:
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java?rev=1614534&r1=1614533&r2=1614534&view=diff
==============================================================================
---
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
(original)
+++
hive/trunk/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitioned.java
Wed Jul 30 01:22:46 2014
@@ -49,10 +49,10 @@ public class TestHCatPartitioned extends
private static List<HCatRecord> writeRecords;
private static List<HCatFieldSchema> partitionColumns;
- @BeforeClass
- public static void oneTimeSetUp() throws Exception {
-
- tableName = "testHCatPartitionedTable";
+ public TestHCatPartitioned(String formatName, String serdeClass, String
inputFormatClass,
+ String outputFormatClass) throws Exception {
+ super(formatName, serdeClass, inputFormatClass, outputFormatClass);
+ tableName = "testHCatPartitionedTable_" + formatName;
writeRecords = new ArrayList<HCatRecord>();
for (int i = 0; i < 20; i++) {
@@ -68,7 +68,6 @@ public class TestHCatPartitioned extends
partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new
FieldSchema("c2", serdeConstants.STRING_TYPE_NAME, "")));
}
-
@Override
protected List<FieldSchema> getPartitionKeys() {
List<FieldSchema> fields = new ArrayList<FieldSchema>();
Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java?rev=1614534&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java
(added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java Wed
Jul 30 01:22:46 2014
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.ql.io.RCFileStorageFormatDescriptor;
+import org.apache.hadoop.hive.ql.io.StorageFormatDescriptor;
+import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat;
+import org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat;
+import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
+import org.apache.hadoop.hive.serde2.avro.AvroSerDe;
+import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Utility class for enumerating Hive native storage formats for testing.
Native Storage formats
+ * are registered via {@link
org.apache.hadoop.hive.ql.io.StorageFormatDescriptor}.
+ */
+public class StorageFormats {
+ /**
+ * Table of additional storage formats. These are SerDes or combinations of
SerDe with
+ * InputFormat and OutputFormat that are not registered as a native Hive
storage format.
+ *
+ * Each row in this table has the following fields:
+ * - formatName - A string name for the storage format. This is used to
give the table created
+ * for the test a unique name.
+ * - serdeClass - The name of the SerDe class used by the storage format.
+ * - inputFormatClass - The name of the InputFormat class.
+ * - outputFormatClass - The name of the OutputFormat class.
+ */
+ public static final Object[][] ADDITIONAL_STORAGE_FORMATS = new Object[][] {
+ {
+ "rcfile_columnar",
+ ColumnarSerDe.class.getName(),
+ RCFileInputFormat.class.getName(),
+ RCFileOutputFormat.class.getName(),
+ }
+ };
+
+ /**
+ * Create an array of Objects used to populate the test paramters.
+ *
+ * @param name Name of the storage format.
+ * @param serdeClass Name of the SerDe class.
+ * @param inputFormatClass Name of the InputFormat class.
+ * @param outputFormatClass Name of the OutputFormat class.
+ * @return Object array containing the arguments.
+ */
+ protected static Object[] createTestArguments(String name, String serdeClass,
+ String inputFormatClass, String outputFormatClass) {
+ Object[] args = {
+ name,
+ serdeClass,
+ inputFormatClass,
+ outputFormatClass
+ };
+ return args;
+ }
+
+ /**
+ * Generates a collection of parameters that can be used as paramters for a
JUnit test fixture.
+ * Each parameter represents one storage format that the fixture will run
against. The list
+ * includes both native Hive storage formats as well as those enumerated in
the
+ * ADDITIONAL_STORAGE_FORMATS table.
+ *
+ * @return List of storage format as paramters.
+ */
+ public static Collection<Object[]> asParameters() {
+ List<Object[]> parameters = new ArrayList<Object[]>();
+
+ // Add test parameters from official storage formats registered with Hive
via
+ // StorageFormatDescriptor.
+ final Configuration conf = new Configuration();
+ for (StorageFormatDescriptor descriptor :
ServiceLoader.load(StorageFormatDescriptor.class)) {
+ String serdeClass = descriptor.getSerde();
+ if (serdeClass == null) {
+ if (descriptor instanceof RCFileStorageFormatDescriptor) {
+ serdeClass = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVEDEFAULTRCFILESERDE);
+ } else {
+ serdeClass = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVEDEFAULTSERDE);
+ }
+ }
+
+ String[] names = new String[descriptor.getNames().size()];
+ names = descriptor.getNames().toArray(names);
+ Object[] arguments = createTestArguments(names[0], serdeClass,
descriptor.getInputFormat(),
+ descriptor.getOutputFormat());
+ parameters.add(arguments);
+ }
+
+ // Add test parameters from storage formats specified in
ADDITIONAL_STORAGE_FORMATS table.
+ for (int i = 0; i < ADDITIONAL_STORAGE_FORMATS.length; i++) {
+ String serdeClass = (String) ADDITIONAL_STORAGE_FORMATS[i][1];
+ String name = (String) ADDITIONAL_STORAGE_FORMATS[i][0];
+ String inputFormatClass = (String) ADDITIONAL_STORAGE_FORMATS[i][2];
+ String outputFormatClass = (String) ADDITIONAL_STORAGE_FORMATS[i][3];
+ assertTrue("InputFormat for storage format not set", inputFormatClass !=
null);
+ assertTrue("OutputFormat for storage format not set", outputFormatClass
!= null);
+ Object[] arguments = createTestArguments(name, serdeClass,
inputFormatClass,
+ outputFormatClass);
+ parameters.add(arguments);
+ }
+
+ return parameters;
+ }
+}
+
