Author: khorgath Date: Tue Mar 25 21:35:52 2014 New Revision: 1581536 URL: http://svn.apache.org/r1581536 Log: HIVE-6724 : HCatStorer throws ClassCastException while storing tinyint/smallint data (Eugene Koifman via Sushanth Sowmyan) (backport from 0.14 trunk)
Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java?rev=1581536&r1=1581535&r2=1581536&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java (original) +++ hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java Tue Mar 25 21:35:52 2014 @@ -149,9 +149,6 @@ public class HCatStorer extends HCatBase */ @Override public void setStoreLocation(String location, Job job) throws IOException { - HCatContext.INSTANCE.setConf(job.getConfiguration()).getConf().get() - .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false); - Configuration config = job.getConfiguration(); config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign); Properties udfProps = UDFContext.getUDFContext().getUDFProperties( Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1581536&r1=1581535&r2=1581536&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java (original) +++ hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java Tue Mar 25 21:35:52 2014 @@ -382,6 +382,10 @@ class PigHCatUtil { * @return object in Pig value system */ public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception { + /*Note that HCatRecordSerDe.serializePrimitiveField() will be called before this, thus some + * type promotion/conversion may occur: e.g. Short to Integer. We should refactor this so + * that it's hapenning in one place per module/product that we are integrating with. + * All Pig conversion should be done here, etc.*/ if(o == null) { return null; } Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1581536&r1=1581535&r2=1581536&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original) +++ hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Tue Mar 25 21:35:52 2014 @@ -94,7 +94,7 @@ public class TestHCatLoader { dropTable(tablename, driver); } static void dropTable(String tablename, Driver driver) throws IOException, CommandNeedRetryException { - driver.run("drop table " + tablename); + driver.run("drop table if exists " + tablename); } private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java?rev=1581536&r1=1581535&r2=1581536&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java (original) +++ hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java Tue Mar 25 21:35:52 2014 @@ -30,8 +30,11 @@ import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.junit.Assert; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -40,8 +43,62 @@ import java.util.List; * please consider @{link TestHCatLoader} or @{link TestHCatStorer}. */ public class TestHCatLoaderStorer extends HCatBaseTest { + private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderStorer.class); /** + * Test round trip of smallint/tinyint: Hive->Pig->Hive. This is a more general use case in HCatalog: + * 'read some data from Hive, process it in Pig, write result back to a Hive table' + */ + @Test + public void testReadWrite() throws Exception { + final String tblName = "small_ints_table"; + final String tblName2 = "pig_hcatalog_1"; + File dataDir = new File(TEST_DATA_DIR + File.separator + "testReadWrite"); + FileUtil.fullyDelete(dataDir); // Might not exist + Assert.assertTrue(dataDir.mkdir()); + final String INPUT_FILE_NAME = dataDir + "/inputtrw.data"; + + TestHCatLoader.dropTable(tblName, driver); + HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, new String[]{"40\t1"}); + + TestHCatLoader.executeStatementOnDriver("create external table " + tblName + + " (my_small_int smallint, my_tiny_int tinyint)" + + " row format delimited fields terminated by '\t' stored as textfile location '" + + dataDir + "'", driver); + TestHCatLoader.dropTable(tblName2, driver); + TestHCatLoader.createTable(tblName2, "my_small_int smallint, my_tiny_int tinyint", null, driver, + "textfile"); + + LOG.debug("File=" + INPUT_FILE_NAME); + TestHCatStorer.dumpFile(INPUT_FILE_NAME); + PigServer server = createPigServer(true); + try { + int queryNumber = 1; + logAndRegister(server, + "A = load '" + tblName + + "' using org.apache.hive.hcatalog.pig.HCatLoader() as (my_small_int:int, my_tiny_int:int);", + queryNumber++); + logAndRegister(server, + "b = foreach A generate my_small_int + my_tiny_int as my_small_int, my_tiny_int;", + queryNumber++); + logAndRegister(server, "store b into '" + tblName2 + + "' using org.apache.hive.hcatalog.pig.HCatStorer();", queryNumber); + //perform simple checksum here; make sure nothing got turned to NULL + TestHCatLoader.executeStatementOnDriver("select my_small_int from " + tblName2, driver); + ArrayList l = new ArrayList(); + driver.getResults(l); + for(Object t : l) { + LOG.debug("t=" + t); + } + Assert.assertEquals("Expected '1' rows; got '" + l.size() + "'", 1, l.size()); + int result = Integer.parseInt((String)l.get(0)); + Assert.assertEquals("Expected value '41'; got '" + result + "'", 41, result); + } + finally { + server.shutdown(); + } + } + /** * Ensure Pig can read/write tinyint/smallint columns. */ @Test @@ -137,5 +194,5 @@ public class TestHCatLoaderStorer extend "store data into 'test_tbl' using org.apache.hive.hcatalog.pig.HCatStorer('','','-onOutOfRangeValue Throw');"); List<ExecJob> jobs = server.executeBatch(); Assert.assertEquals(expectedStatus, jobs.get(0).getStatus()); - } + } } Modified: hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1581536&r1=1581535&r2=1581536&view=diff ============================================================================== --- hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original) +++ hive/branches/branch-0.13/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Tue Mar 25 21:35:52 2014 @@ -358,7 +358,7 @@ public class TestHCatStorer extends HCat } Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead); } - private static void dumpFile(String fileName) throws Exception { + static void dumpFile(String fileName) throws Exception { File file = new File(fileName); BufferedReader reader = new BufferedReader(new FileReader(file)); String line = null;