Author: avandana
Date: Fri Dec 21 23:52:56 2012
New Revision: 1425202
URL: http://svn.apache.org/viewvc?rev=1425202&view=rev
Log:
HCAT-567 HCatClient must allow retrieval of multiple partitions using a partial
partition spec.(for branch)
Modified:
incubator/hcatalog/branches/branch-0.4/CHANGES.txt
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1425202&r1=1425201&r2=1425202&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Fri Dec 21 23:52:56 2012
@@ -71,6 +71,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-567 HCatClient must allow retrieval of multiple partitions using a
partial partition spec.(mithun via avandana)
+
HCAT-554 Loading data using HCatLoader() from a table on non default
namenode fails.(amalakar via avandana)
HCAT-566 HCatTable doesn't report partition columns correctly. (mithun via
traviscrawford)
Modified:
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java?rev=1425202&r1=1425201&r2=1425202&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClient.java
Fri Dec 21 23:52:56 2012
@@ -204,11 +204,30 @@ public abstract class HCatClient {
throws HCatException;
/**
+ * Gets all the partitions that match the specified (and possibly partial)
partition specification.
+ * A partial partition-specification is one where not all partition-keys
have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region
string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions
('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: getPartitions('myDb', 'myTable',
{dt='20120101', region='us'}) would return 1 partition.
+ * 2. Partial partition spec: getPartitions('myDb', 'myTable',
{dt='20120101'}) would return all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us',
'uk' and 'in').
+ * @param dbName The name of the database.
+ * @param tableName The name of the table.
+ * @param partitionSpec The partition specification. (Need not include all
partition keys.)
+ * @return A list of partitions.
+ * @throws HCatException
+ */
+ public abstract List<HCatPartition> getPartitions(String dbName, String
tableName,
+ Map<String,String>
partitionSpec) throws HCatException;
+
+
+ /**
* Gets the partition.
*
* @param dbName The database name.
* @param tableName The table name.
- * @param partitionSpec The partition specification,
{[col_name,value],[col_name2,value2]}.
+ * @param partitionSpec The partition specification,
{[col_name,value],[col_name2,value2]}. All partition-key-values
+ * must be specified.
* @return An instance of HCatPartitionInfo.
* @throws HCatException
*/
@@ -235,15 +254,20 @@ public abstract class HCatClient {
throws HCatException;
/**
- * Drops partition.
- *
+ * Drops partition(s) that match the specified (and possibly partial)
partition specification.
+ * A partial partition-specification is one where not all partition-keys
have associated values. For example,
+ * for a table ('myDb.myTable') with 2 partition keys (dt string, region
string),
+ * if for each dt ('20120101', '20120102', etc.) there can exist 3 regions
('us', 'uk', 'in'), then,
+ * 1. Complete partition spec: dropPartitions('myDb', 'myTable',
{dt='20120101', region='us'}) would drop 1 partition.
+ * 2. Partial partition spec: dropPartitions('myDb', 'myTable',
{dt='20120101'}) would drop all 3 partitions,
+ * with dt='20120101' (i.e. region = 'us',
'uk' and 'in').
* @param dbName The database name.
* @param tableName The table name.
* @param partitionSpec The partition specification,
{[col_name,value],[col_name2,value2]}.
* @param ifExists Hive returns an error if the partition specified does
not exist, unless ifExists is set to true.
* @throws HCatException
*/
- public abstract void dropPartition(String dbName, String tableName,
+ public abstract void dropPartitions(String dbName, String tableName,
Map<String, String> partitionSpec, boolean ifExists)
throws HCatException;
Modified:
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java?rev=1425202&r1=1425201&r2=1425202&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/main/java/org/apache/hcatalog/api/HCatClientHMSImpl.java
Fri Dec 21 23:52:56 2012
@@ -327,12 +327,44 @@ public class HCatClientHMSImpl extends H
}
@Override
+ public List<HCatPartition> getPartitions(String dbName, String tblName,
Map<String, String> partitionSpec) throws HCatException {
+ return listPartitionsByFilter(dbName, tblName,
getFilterString(partitionSpec));
+ }
+
+ private static String getFilterString(Map<String, String> partitionSpec) {
+ final String AND = " AND ";
+ StringBuilder filter = new StringBuilder();
+ for (Map.Entry<String, String> entry : partitionSpec.entrySet()) {
+
filter.append(entry.getKey()).append("=").append("\"").append(entry.getValue()).append("\"").append(AND);
+ }
+
+ int length = filter.toString().length();
+ if (length > 0)
+ filter.delete(length - AND.length(), length);
+
+ return filter.toString();
+ }
+
+ @Override
public HCatPartition getPartition(String dbName, String tableName,
Map<String, String> partitionSpec) throws HCatException {
HCatPartition partition = null;
try {
+ List<HCatFieldSchema> partitionColumns = getTable(checkDB(dbName),
tableName).getPartCols();
+ if (partitionColumns.size() != partitionSpec.size()) {
+ throw new HCatException("Partition-spec doesn't have the right
number of partition keys.");
+ }
+
ArrayList<String> ptnValues = new ArrayList<String>();
- ptnValues.addAll(partitionSpec.values());
+ for (HCatFieldSchema partitionColumn : partitionColumns) {
+ String partKey = partitionColumn.getName();
+ if (partitionSpec.containsKey(partKey)) {
+ ptnValues.add(partitionSpec.get(partKey)); //
Partition-keys added in order.
+ }
+ else {
+ throw new HCatException("Invalid partition-key specified:
" + partKey);
+ }
+ }
Partition hivePartition = hmsClient.getPartition(checkDB(dbName),
tableName, ptnValues);
if (hivePartition != null) {
@@ -383,19 +415,22 @@ public class HCatClientHMSImpl extends H
}
@Override
- public void dropPartition(String dbName, String tableName,
+ public void dropPartitions(String dbName, String tableName,
Map<String, String> partitionSpec, boolean ifExists)
throws HCatException {
try {
- List<String> ptnValues = new ArrayList<String>();
- ptnValues.addAll(partitionSpec.values());
- hmsClient.dropPartition(checkDB(dbName), tableName, ptnValues,
- ifExists);
- } catch (NoSuchObjectException e) {
- if (!ifExists) {
- throw new ObjectNotFoundException(
- "NoSuchObjectException while dropping partition.", e);
+ dbName = checkDB(dbName);
+ List<Partition> partitions =
hmsClient.listPartitionsByFilter(dbName, tableName,
+ getFilterString(partitionSpec), (short)-1);
+
+ for (Partition partition : partitions) {
+ dropPartition(partition, ifExists);
}
+
+ } catch (NoSuchObjectException e) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition. " +
+ "Either db(" + dbName + ") or table(" + tableName
+ ") missing.", e);
} catch (MetaException e) {
throw new HCatException("MetaException while dropping partition.",
e);
@@ -405,6 +440,18 @@ public class HCatClientHMSImpl extends H
}
}
+ private void dropPartition(Partition partition, boolean ifExists)
+ throws HCatException, MetaException, TException {
+ try {
+ hmsClient.dropPartition(partition.getDbName(),
partition.getTableName(), partition.getValues());
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ObjectNotFoundException(
+ "NoSuchObjectException while dropping partition: " +
partition.getValues(), e);
+ }
+ }
+ }
+
@Override
public List<HCatPartition> listPartitionsByFilter(String dbName,
String tblName, String filter) throws HCatException {
Modified:
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1425202&r1=1425201&r2=1425202&view=diff
==============================================================================
---
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
(original)
+++
incubator/hcatalog/branches/branch-0.4/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
Fri Dec 21 23:52:56 2012
@@ -215,7 +215,7 @@ public class TestHCatClient {
HCatPartition ptn = client.getPartition(dbName, tableName, firstPtn);
assertTrue(ptn != null);
- client.dropPartition(dbName, tableName, firstPtn, true);
+ client.dropPartitions(dbName, tableName, firstPtn, true);
ptnList = client.listPartitionsByFilter(dbName,
tableName, null);
assertTrue(ptnList.size() == 2);
@@ -503,6 +503,104 @@ public class TestHCatClient {
}
@Test
+ public void testGetPartitionsWithPartialSpec() throws Exception {
+ try {
+ HCatClient client = HCatClient.create(new Configuration(hcatConf));
+ final String dbName = "myDb";
+ final String tableName = "myTable";
+
+ client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ client.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ List<HCatFieldSchema> columnSchema = Arrays.asList(new
HCatFieldSchema("foo", Type.INT, ""),
+ new HCatFieldSchema("bar", Type.STRING, ""));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(new
HCatFieldSchema("dt", Type.STRING, ""),
+ new HCatFieldSchema("grid", Type.STRING, ""));
+
+ client.createTable(HCatCreateTableDesc.create(dbName, tableName,
columnSchema).partCols(new
ArrayList<HCatFieldSchema>(partitionSchema)).build());
+
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2011_12_31");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2012_01_01");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "OB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "XB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+
+ Map<String, String> partialPartitionSpec = new HashMap<String,
String>();
+ partialPartitionSpec.put("dt", "2012_01_01");
+
+ List<HCatPartition> partitions = client.getPartitions(dbName,
tableName, partialPartitionSpec);
+ assertEquals("Unexpected number of partitions.", 3,
partitions.size());
+ assertArrayEquals("Mismatched partition.", new
String[]{"2012_01_01", "AB"}, partitions.get(0).getValues().toArray());
+ assertArrayEquals("Mismatched partition.", new
String[]{"2012_01_01", "OB"}, partitions.get(1).getValues().toArray());
+ assertArrayEquals("Mismatched partition.", new
String[]{"2012_01_01", "XB"}, partitions.get(2).getValues().toArray());
+
+ client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE);
+ }
+ catch (Exception unexpected) {
+ LOG.error("Unexpected exception!", unexpected);
+ assertTrue("Unexpected exception! " + unexpected.getMessage(),
false);
+ }
+ }
+
+ @Test
+ public void testDropPartitionsWithPartialSpec() throws Exception {
+ try {
+ HCatClient client = HCatClient.create(new Configuration(hcatConf));
+ final String dbName = "myDb";
+ final String tableName = "myTable";
+
+ client.dropDatabase(dbName, true, HCatClient.DropDBMode.CASCADE);
+
+ client.createDatabase(HCatCreateDBDesc.create(dbName).build());
+ List<HCatFieldSchema> columnSchema = Arrays.asList(new
HCatFieldSchema("foo", Type.INT, ""),
+ new HCatFieldSchema("bar", Type.STRING, ""));
+
+ List<HCatFieldSchema> partitionSchema = Arrays.asList(new
HCatFieldSchema("dt", Type.STRING, ""),
+ new HCatFieldSchema("grid", Type.STRING, ""));
+
+ client.createTable(HCatCreateTableDesc.create(dbName, tableName,
columnSchema).partCols(new
ArrayList<HCatFieldSchema>(partitionSchema)).build());
+
+ Map<String, String> partitionSpec = new HashMap<String, String>();
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2011_12_31");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("grid", "AB");
+ partitionSpec.put("dt", "2012_01_01");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "OB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+ partitionSpec.put("dt", "2012_01_01");
+ partitionSpec.put("grid", "XB");
+ client.addPartition(HCatAddPartitionDesc.create(dbName, tableName,
"", partitionSpec).build());
+
+ Map<String, String> partialPartitionSpec = new HashMap<String,
String>();
+ partialPartitionSpec.put("dt", "2012_01_01");
+
+ client.dropPartitions(dbName, tableName, partialPartitionSpec,
true);
+
+ List<HCatPartition> partitions = client.getPartitions(dbName,
tableName);
+ assertEquals("Unexpected number of partitions.", 1,
partitions.size());
+ assertArrayEquals("Mismatched partition.", new
String[]{"2011_12_31", "AB"}, partitions.get(0).getValues().toArray());
+
+ client.dropDatabase(dbName, false, HCatClient.DropDBMode.CASCADE);
+ }
+ catch (Exception unexpected) {
+ LOG.error("Unexpected exception!", unexpected);
+ assertTrue("Unexpected exception! " + unexpected.getMessage(),
false);
+ }
+ }
+
+ @Test
public void testPartitionSchema() throws Exception {
try {
HCatClient client = HCatClient.create(new Configuration(hcatConf));
