Author: hashutosh
Date: Tue Jan 24 03:32:43 2012
New Revision: 1235120
URL: http://svn.apache.org/viewvc?rev=1235120&view=rev
Log:
HCATALOG-227 : NPE on HBaseInputStorageDriver when table contains a CF that's
never been successfully written to (toffer via hashutosh)
Modified:
incubator/hcatalog/branches/branch-0.3/CHANGES.txt
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
Modified: incubator/hcatalog/branches/branch-0.3/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/CHANGES.txt?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.3/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.3/CHANGES.txt Tue Jan 24 03:32:43 2012
@@ -105,6 +105,8 @@ Release 0.3.0 (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-227. NPE on HBaseInputStorageDriver when table contains a CF that's
never been successfully written to (toffer via hashutosh)
+
HCAT-226. HBase storage drivers should load hbase resources (hbase-site,
hbase-default) for the user (toffer via hashutosh)
HCAT-224. hcatalog e2e tests have the hive version hard coded we should read
that from the hive build.properties (Arpit Gupta via hashutosh)
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HBaseInputStorageDriver.java
Tue Jan 24 03:32:43 2012
@@ -221,7 +221,7 @@ public class HBaseInputStorageDriver ext
}
HCatTableSnapshot hcatSnapshot = new HCatTableSnapshot(
- hcatTableInfo.getDatabaseName(),
hcatTableInfo.getTableName(),revisionMap);
+ hcatTableInfo.getDatabaseName(),
hcatTableInfo.getTableName(),revisionMap,hbaseSnapshot.getLatestRevision());
return hcatSnapshot;
}
@@ -241,7 +241,7 @@ public class HBaseInputStorageDriver ext
String fullyQualifiedName = hcatSnapshot.getDatabaseName() + "."
+ hcatSnapshot.getTableName();
- return new TableSnapshot(fullyQualifiedName, revisionMap);
+ return new TableSnapshot(fullyQualifiedName,
revisionMap,hcatSnapshot.getLatestRevision());
}
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/HCatTableSnapshot.java
Tue Jan 24 03:32:43 2012
@@ -33,11 +33,13 @@ public class HCatTableSnapshot implement
private String tableName;
private String databaseName;
private Map<String, Long> columnMap;
+ private long latestRevision;
- HCatTableSnapshot(String databaseName, String tableName, Map<String, Long>
columnMap) {
+ HCatTableSnapshot(String databaseName, String tableName, Map<String, Long>
columnMap, long latestRevision) {
this.tableName = tableName;
this.databaseName = databaseName;
this.columnMap = columnMap;
+ this.latestRevision = latestRevision;
}
/**
@@ -58,7 +60,9 @@ public class HCatTableSnapshot implement
* @return The revision number of a column in a snapshot.
*/
long getRevision(String column){
- return this.columnMap.get(column);
+ if(columnMap.containsKey(column))
+ return this.columnMap.get(column);
+ return latestRevision;
}
/**
@@ -71,10 +75,17 @@ public class HCatTableSnapshot implement
return this.columnMap.containsKey(column);
}
+ /**
+ * @return latest committed revision when snapshot was taken
+ */
+ long getLatestRevision() {
+ return latestRevision;
+ }
+
@Override
public String toString() {
String snapshot = " Database Name: " + this.databaseName +" Table Name
: " + tableName +
- " Column revision : " + columnMap.toString();
+ "Latest Revision: "+latestRevision+" Column revision : " +
columnMap.toString();
return snapshot;
}
}
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/TableSnapshot.java
Tue Jan 24 03:32:43 2012
@@ -30,10 +30,13 @@ public class TableSnapshot {
private Map<String, Long> cfRevisionMap;
+ private long latestRevision;
- public TableSnapshot(String name, Map<String, Long> cfRevMap) {
+
+ public TableSnapshot(String name, Map<String, Long> cfRevMap, long
latestRevision) {
this.name = name;
this.cfRevisionMap = cfRevMap;
+ this.latestRevision = latestRevision;
}
/**
@@ -61,12 +64,21 @@ public class TableSnapshot {
* @return the revision
*/
public long getRevision(String familyName){
- return this.cfRevisionMap.get(familyName);
+ if(cfRevisionMap.containsKey(familyName))
+ return cfRevisionMap.get(familyName);
+ return latestRevision;
+ }
+
+ /**
+ * @return the latest committed revision when this snapshot was taken
+ */
+ public long getLatestRevision() {
+ return latestRevision;
}
@Override
public String toString() {
- String snapshot = "Table Name : " + name
+ String snapshot = "Table Name : " + name +" Latest Revision: " +
latestRevision
+ " Column Familiy revision : " + cfRevisionMap.toString();
return snapshot;
}
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/java/org/apache/hcatalog/hbase/snapshot/ZKBasedRevisionManager.java
Tue Jan 24 03:32:43 2012
@@ -320,7 +320,9 @@ public class ZKBasedRevisionManager impl
cfMap.put(cfName, version);
}
- return new TableSnapshot(tableName, cfMap);
+ TableSnapshot snapshot = new TableSnapshot(tableName, cfMap,latestID);
+ LOG.debug("Created snapshot For table: "+tableName+" snapshot:
"+snapshot);
+ return snapshot;
}
/* This method allows the user to create snapshot of a
@@ -346,7 +348,7 @@ public class ZKBasedRevisionManager impl
cfMap.put(cf, revision);
}
- return new TableSnapshot(tableName, cfMap);
+ return new TableSnapshot(tableName, cfMap, revision);
}
/**
Modified:
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java?rev=1235120&r1=1235119&r2=1235120&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
(original)
+++
incubator/hcatalog/branches/branch-0.3/storage-drivers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java
Tue Jan 24 03:32:43 2012
@@ -81,7 +81,7 @@ public class TestSnapshots extends Skele
Map<String, Long> revMap = new HashMap<String, Long>();
revMap.put("cf1", 3L);
revMap.put("cf2", 5L);
- TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName,
revMap);
+ TableSnapshot hbaseSnapshot = new TableSnapshot(fullyQualTableName,
revMap,-1);
HCatTableSnapshot hcatSnapshot =
HBaseInputStorageDriver.convertSnapshot(hbaseSnapshot,
inputInfo.getTableInfo());
assertEquals(hcatSnapshot.getRevision("value1"), 3);
@@ -101,7 +101,7 @@ public class TestSnapshots extends Skele
assertEquals(0, cmdResponse.getResponseCode());
revMap.clear();
revMap.put("cf1", 3L);
- hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap);
+ hbaseSnapshot = new TableSnapshot(fullyQualTableName, revMap, -1);
inputInfo = InputJobInfo.create(databaseName, tableName, null, null,
null);
InitializeInput.setInput(job, inputInfo);
modifiedInputInfo =
job.getConfiguration().get(HCatConstants.HCAT_KEY_JOB_INFO);
