Author: khorgath
Date: Tue Mar 25 21:25:17 2014
New Revision: 1581531
URL: http://svn.apache.org/r1581531
Log:
HIVE-6724 : HCatStorer throws ClassCastException while storing tinyint/smallint
data (Eugene Koifman via Sushanth Sowmyan)
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
Tue Mar 25 21:25:17 2014
@@ -149,9 +149,6 @@ public class HCatStorer extends HCatBase
*/
@Override
public void setStoreLocation(String location, Job job) throws IOException {
- HCatContext.INSTANCE.setConf(job.getConfiguration()).getConf().get()
- .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
-
Configuration config = job.getConfiguration();
config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign);
Properties udfProps = UDFContext.getUDFContext().getUDFProperties(
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
Tue Mar 25 21:25:17 2014
@@ -381,6 +381,10 @@ class PigHCatUtil {
* @return object in Pig value system
*/
public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws
Exception {
+ /*Note that HCatRecordSerDe.serializePrimitiveField() will be called
before this, thus some
+ * type promotion/conversion may occur: e.g. Short to Integer. We should
refactor this so
+ * that it's hapenning in one place per module/product that we are
integrating with.
+ * All Pig conversion should be done here, etc.*/
if(o == null) {
return null;
}
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
Tue Mar 25 21:25:17 2014
@@ -94,7 +94,7 @@ public class TestHCatLoader {
dropTable(tablename, driver);
}
static void dropTable(String tablename, Driver driver) throws IOException,
CommandNeedRetryException {
- driver.run("drop table " + tablename);
+ driver.run("drop table if exists " + tablename);
}
private void createTable(String tablename, String schema, String
partitionedBy) throws IOException, CommandNeedRetryException {
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
Tue Mar 25 21:25:17 2014
@@ -30,8 +30,11 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -40,8 +43,62 @@ import java.util.List;
* please consider @{link TestHCatLoader} or @{link TestHCatStorer}.
*/
public class TestHCatLoaderStorer extends HCatBaseTest {
+ private static final Logger LOG =
LoggerFactory.getLogger(TestHCatLoaderStorer.class);
/**
+ * Test round trip of smallint/tinyint: Hive->Pig->Hive. This is a more
general use case in HCatalog:
+ * 'read some data from Hive, process it in Pig, write result back to a Hive
table'
+ */
+ @Test
+ public void testReadWrite() throws Exception {
+ final String tblName = "small_ints_table";
+ final String tblName2 = "pig_hcatalog_1";
+ File dataDir = new File(TEST_DATA_DIR + File.separator + "testReadWrite");
+ FileUtil.fullyDelete(dataDir); // Might not exist
+ Assert.assertTrue(dataDir.mkdir());
+ final String INPUT_FILE_NAME = dataDir + "/inputtrw.data";
+
+ TestHCatLoader.dropTable(tblName, driver);
+ HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, new String[]{"40\t1"});
+
+ TestHCatLoader.executeStatementOnDriver("create external table " + tblName
+
+ " (my_small_int smallint, my_tiny_int tinyint)" +
+ " row format delimited fields terminated by '\t' stored as textfile
location '" +
+ dataDir + "'", driver);
+ TestHCatLoader.dropTable(tblName2, driver);
+ TestHCatLoader.createTable(tblName2, "my_small_int smallint, my_tiny_int
tinyint", null, driver,
+ "textfile");
+
+ LOG.debug("File=" + INPUT_FILE_NAME);
+ TestHCatStorer.dumpFile(INPUT_FILE_NAME);
+ PigServer server = createPigServer(true);
+ try {
+ int queryNumber = 1;
+ logAndRegister(server,
+ "A = load '" + tblName +
+ "' using org.apache.hive.hcatalog.pig.HCatLoader() as
(my_small_int:int, my_tiny_int:int);",
+ queryNumber++);
+ logAndRegister(server,
+ "b = foreach A generate my_small_int + my_tiny_int as my_small_int,
my_tiny_int;",
+ queryNumber++);
+ logAndRegister(server, "store b into '" + tblName2 +
+ "' using org.apache.hive.hcatalog.pig.HCatStorer();", queryNumber);
+ //perform simple checksum here; make sure nothing got turned to NULL
+ TestHCatLoader.executeStatementOnDriver("select my_small_int from " +
tblName2, driver);
+ ArrayList l = new ArrayList();
+ driver.getResults(l);
+ for(Object t : l) {
+ LOG.debug("t=" + t);
+ }
+ Assert.assertEquals("Expected '1' rows; got '" + l.size() + "'", 1,
l.size());
+ int result = Integer.parseInt((String)l.get(0));
+ Assert.assertEquals("Expected value '41'; got '" + result + "'", 41,
result);
+ }
+ finally {
+ server.shutdown();
+ }
+ }
+ /**
* Ensure Pig can read/write tinyint/smallint columns.
*/
@Test
@@ -137,5 +194,5 @@ public class TestHCatLoaderStorer extend
"store data into 'test_tbl' using
org.apache.hive.hcatalog.pig.HCatStorer('','','-onOutOfRangeValue Throw');");
List<ExecJob> jobs = server.executeBatch();
Assert.assertEquals(expectedStatus, jobs.get(0).getStatus());
- }
+ }
}
Modified:
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
---
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
(original)
+++
hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
Tue Mar 25 21:25:17 2014
@@ -358,7 +358,7 @@ public class TestHCatStorer extends HCat
}
Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead +
" file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead);
}
- private static void dumpFile(String fileName) throws Exception {
+ static void dumpFile(String fileName) throws Exception {
File file = new File(fileName);
BufferedReader reader = new BufferedReader(new FileReader(file));
String line = null;
