Author: hashutosh
Date: Thu Dec 12 17:59:31 2013
New Revision: 1550461
URL: http://svn.apache.org/r1550461
Log:
HIVE-6018 : FetchTask should not reference metastore classes (Navis via Prasad
Mujumdar)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
---
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
(original)
+++
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
Thu Dec 12 17:59:31 2013
@@ -310,42 +310,6 @@ public class MetaStoreUtils {
/**
* getDeserializer
*
- * Get the Deserializer for a table given its name and properties.
- *
- * @param conf
- * hadoop config
- * @param schema
- * the properties to use to instantiate the deserializer
- * @return
- * Returns instantiated deserializer by looking up class name of
deserializer stored in passed
- * in properties. Also, initializes the deserializer with schema stored in
passed in properties.
- * @exception MetaException
- * if any problems instantiating the Deserializer
- *
- * todo - this should move somewhere into serde.jar
- *
- */
- static public Deserializer getDeserializer(Configuration conf,
- Properties schema) throws MetaException {
- try {
- String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB);
- if(clazzName == null) {
- throw new IllegalStateException("Property " +
serdeConstants.SERIALIZATION_LIB + " cannot be null");
- }
- Deserializer deserializer =
ReflectionUtils.newInstance(conf.getClassByName(clazzName)
- .asSubclass(Deserializer.class), conf);
- deserializer.initialize(conf, schema);
- return deserializer;
- } catch (Exception e) {
- LOG.error("error in initSerDe: " + e.getClass().getName() + " "
- + e.getMessage(), e);
- throw new MetaException(e.getClass().getName() + " " + e.getMessage());
- }
- }
-
- /**
- * getDeserializer
- *
* Get the Deserializer for a table.
*
* @param conf
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
Thu Dec 12 17:59:31 2013
@@ -400,7 +400,7 @@ public class FetchOperator implements Se
this.inputSplits = inputSplits;
splitNum = 0;
- serde = partDesc.getDeserializer();
+ serde = partDesc.getDeserializer(job);
serde.initialize(job, partDesc.getOverlayedProperties());
if (currTbl != null) {
@@ -646,7 +646,7 @@ public class FetchOperator implements Se
// Get the OI corresponding to all the partitions
for (PartitionDesc listPart : listParts) {
partition = listPart;
- Deserializer partSerde = listPart.getDeserializer();
+ Deserializer partSerde = listPart.getDeserializer(job);
partSerde.initialize(job, listPart.getOverlayedProperties());
partitionedTableOI = ObjectInspectorConverters.getConvertedOI(
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveInputFormat.java
Thu Dec 12 17:59:31 2013
@@ -307,8 +307,12 @@ public class CombineHiveInputFormat<K ex
Class inputFormatClass = part.getInputFileFormatClass();
String inputFormatClassName = inputFormatClass.getName();
InputFormat inputFormat = getInputFormatFromCache(inputFormatClass, job);
- String deserializerClassName = part.getDeserializer() == null ? null
- : part.getDeserializer().getClass().getName();
+ String deserializerClassName = null;
+ try {
+ deserializerClassName = part.getDeserializer(job).getClass().getName();
+ } catch (Exception e) {
+ // ignore
+ }
// Since there is no easy way of knowing whether MAPREDUCE-1597 is
present in the tree or not,
// we use a configuration variable for the same
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
Thu Dec 12 17:59:31 2013
@@ -260,19 +260,6 @@ public class Partition implements Serial
return deserializer;
}
- final public Deserializer getDeserializer(Properties props) {
- if (deserializer == null) {
- try {
- deserializer = MetaStoreUtils.getDeserializer(Hive.get().getConf(),
props);
- } catch (HiveException e) {
- throw new RuntimeException(e);
- } catch (MetaException e) {
- throw new RuntimeException(e);
- }
- }
- return deserializer;
- }
-
public Properties getSchema() {
return MetaStoreUtils.getSchema(tPartition, table.getTTable());
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java?rev=1550461&r1=1550460&r2=1550461&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
Thu Dec 12 17:59:31 2013
@@ -23,18 +23,18 @@ import java.util.Enumeration;
import java.util.LinkedHashMap;
import java.util.Properties;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
/**
* PartitionDesc.
@@ -105,14 +105,19 @@ public class PartitionDesc implements Se
}
/**
- * Return a deserializer object corresponding to the tableDesc.
+ * Return a deserializer object corresponding to the partitionDesc.
*/
- public Deserializer getDeserializer() {
- try {
- return MetaStoreUtils.getDeserializer(Hive.get().getConf(),
getProperties());
- } catch (Exception e) {
- return null;
- }
+ public Deserializer getDeserializer(Configuration conf) throws Exception {
+ Properties schema = getProperties();
+ String clazzName = schema.getProperty(serdeConstants.SERIALIZATION_LIB);
+ if (clazzName == null) {
+ throw new IllegalStateException("Property " +
serdeConstants.SERIALIZATION_LIB +
+ " cannot be null");
+ }
+ Deserializer deserializer =
ReflectionUtils.newInstance(conf.getClassByName(clazzName)
+ .asSubclass(Deserializer.class), conf);
+ deserializer.initialize(conf, schema);
+ return deserializer;
}
public void setInputFileFormatClass(
