Author: toffer
Date: Fri Mar 9 22:38:02 2012
New Revision: 1299082
URL: http://svn.apache.org/viewvc?rev=1299082&view=rev
Log:
Merged from Trunk: HCATALOG-303 Fix build failure due to
HBaseSerDe.parseColumnMapping method signature change in hive (rohini via
toffer)
Added:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java
- copied unchanged from r1299080,
incubator/hcatalog/trunk/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseUtil.java
Removed:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseMapredUtil.java
Modified:
incubator/hcatalog/branches/branch-0.4/ (props changed)
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
Propchange: incubator/hcatalog/branches/branch-0.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Mar 9 22:38:02 2012
@@ -1 +1 @@
-/incubator/hcatalog/trunk:1296568
+/incubator/hcatalog/trunk:1296568,1299080
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Fri Mar 9 22:38:02 2012
@@ -66,6 +66,8 @@ Release 0.4.0 - Unreleased
OPTIMIZATIONS
BUG FIXES
+ HCAT-303 Fix build failure due to HBaseSerDe.parseColumnMapping method
signature change in hive (rohini via toffer)
+
HCAT-282 HCatInputFormat shouldn't expect storageHandler to be serializable
(khorgath via gates)
HCAT-281 HCat use table schema instead of partition schema to read partition
data (daijy and khorgath via gates)
Modified:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
Fri Mar 9 22:38:02 2012
@@ -65,7 +65,7 @@ class HBaseBulkOutputFormat extends HBas
job.setOutputValueClass(Put.class);
job.setOutputCommitter(HBaseBulkOutputCommitter.class);
baseOutputFormat.checkOutputSpecs(ignored, job);
- HBaseMapredUtil.addHBaseDelegationToken(job);
+ HBaseUtil.addHBaseDelegationToken(job);
addJTDelegationToken(job);
}
Modified:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseDirectOutputFormat.java
Fri Mar 9 22:38:02 2012
@@ -67,7 +67,7 @@ class HBaseDirectOutputFormat extends HB
job.setIfUnset(TableOutputFormat.OUTPUT_TABLE,
job.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY));
outputFormat.checkOutputSpecs(ignored, job);
- HBaseMapredUtil.addHBaseDelegationToken(job);
+ HBaseUtil.addHBaseDelegationToken(job);
}
private static class HBaseDirectRecordWriter implements
Modified:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseHCatStorageHandler.java
Fri Mar 9 22:38:02 2012
@@ -244,10 +244,9 @@ public class HBaseHCatStorageHandler ext
List<String> hbaseColumnFamilies = new ArrayList<String>();
List<String> hbaseColumnQualifiers = new ArrayList<String>();
List<byte[]> hbaseColumnFamiliesBytes = new ArrayList<byte[]>();
- List<byte[]> hbaseColumnQualifiersBytes = new ArrayList<byte[]>();
- int iKey = HBaseSerDe.parseColumnMapping(hbaseColumnsMapping,
+ int iKey = HBaseUtil.parseColumnMapping(hbaseColumnsMapping,
hbaseColumnFamilies, hbaseColumnFamiliesBytes,
- hbaseColumnQualifiers, hbaseColumnQualifiersBytes);
+ hbaseColumnQualifiers, null);
HTableDescriptor tableDesc;
Set<String> uniqueColumnFamilies = new HashSet<String>();
@@ -313,8 +312,8 @@ public class HBaseHCatStorageHandler ext
throw new MetaException(StringUtils.stringifyException(mnre));
} catch (IOException ie) {
throw new MetaException(StringUtils.stringifyException(ie));
- } catch (SerDeException se) {
- throw new MetaException(StringUtils.stringifyException(se));
+ } catch (IllegalArgumentException iae) {
+ throw new MetaException(StringUtils.stringifyException(iae));
}
}
@@ -556,27 +555,23 @@ public class HBaseHCatStorageHandler ext
int position = tableSchema.getPosition(fieldName);
outputColumnMapping.add(position);
}
- try {
- List<String> columnFamilies = new ArrayList<String>();
- List<String> columnQualifiers = new ArrayList<String>();
- HBaseSerDe.parseColumnMapping(hbaseColumnMapping,
columnFamilies, null,
- columnQualifiers, null);
- for (int i = 0; i < outputColumnMapping.size(); i++) {
- int cfIndex = outputColumnMapping.get(i);
- String cf = columnFamilies.get(cfIndex);
- // We skip the key column.
- if (cf.equals(HBaseSerDe.HBASE_KEY_COL) == false) {
- String qualifier = columnQualifiers.get(i);
- builder.append(cf);
- builder.append(":");
- if (qualifier != null) {
- builder.append(qualifier);
- }
- builder.append(" ");
+ List<String> columnFamilies = new ArrayList<String>();
+ List<String> columnQualifiers = new ArrayList<String>();
+ HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies,
null,
+ columnQualifiers, null);
+ for (int i = 0; i < outputColumnMapping.size(); i++) {
+ int cfIndex = outputColumnMapping.get(i);
+ String cf = columnFamilies.get(cfIndex);
+ // We skip the key column.
+ if (cf.equals(HBaseSerDe.HBASE_KEY_COL) == false) {
+ String qualifier = columnQualifiers.get(i);
+ builder.append(cf);
+ builder.append(":");
+ if (qualifier != null) {
+ builder.append(qualifier);
}
+ builder.append(" ");
}
- } catch (SerDeException e) {
- throw new IOException(e);
}
}
//Remove the extra space delimiter
Modified:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputFormat.java
Fri Mar 9 22:38:02 2012
@@ -105,7 +105,7 @@ class HBaseInputFormat implements InputF
public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int
numSplits)
throws IOException {
inputFormat.setConf(job);
- HBaseMapredUtil.addHBaseDelegationToken(job);
+ HBaseUtil.addHBaseDelegationToken(job);
return
convertSplits(inputFormat.getSplits(HCatMapRedUtil.createJobContext(job, null,
Reporter.NULL)));
}
Modified:
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java?rev=1299082&r1=1299081&r2=1299082&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseRevisionManagerUtil.java
Fri Mar 9 22:38:02 2012
@@ -32,7 +32,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hive.hbase.HBaseSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hcatalog.common.HCatConstants;
import org.apache.hcatalog.common.HCatUtil;
import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -273,12 +272,8 @@ class HBaseRevisionManagerUtil {
Map<String, String> hcatHbaseColMap = new HashMap<String, String>();
List<String> columnFamilies = new ArrayList<String>();
List<String> columnQualifiers = new ArrayList<String>();
- try {
- HBaseSerDe.parseColumnMapping(hbaseColumnMapping, columnFamilies,
- null, columnQualifiers, null);
- } catch (SerDeException e) {
- throw new IOException("Exception while converting snapshots.", e);
- }
+ HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies,
+ null, columnQualifiers, null);
for (HCatFieldSchema column : hcatTableSchema.getFields()) {
int fieldPos = hcatTableSchema.getPosition(column.getName());
