Author: gates
Date: Wed Feb 5 18:18:28 2014
New Revision: 1564859
URL: http://svn.apache.org/r1564859
Log:
HIVE-6251 Add ability to specify delimiter in HCatalog Java API to create
tables - HCatCreateTableDesc (ekoifman via gates)
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java?rev=1564859&r1=1564858&r2=1564859&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java
(original)
+++
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatCreateTableDesc.java
Wed Feb 5 18:18:28 2014
@@ -19,6 +19,7 @@
package org.apache.hive.hcatalog.api;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hive.ql.io.RCFi
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -72,6 +74,7 @@ public class HCatCreateTableDesc {
private String outputformat;
private String serde;
private String storageHandler;
+ private Map<String, String> serdeParams;
private HCatCreateTableDesc(String dbName, String tableName,
List<HCatFieldSchema> columns) {
this.dbName = dbName;
@@ -146,6 +149,11 @@ public class HCatCreateTableDesc {
}
}
newTable.setSd(sd);
+ if(serdeParams != null) {
+ for(Map.Entry<String, String> param : serdeParams.entrySet()) {
+ sd.getSerdeInfo().putToParameters(param.getKey(), param.getValue());
+ }
+ }
if (this.partCols != null) {
ArrayList<FieldSchema> hivePtnCols = new ArrayList<FieldSchema>();
for (HCatFieldSchema fs : this.partCols) {
@@ -296,6 +304,12 @@ public class HCatCreateTableDesc {
public String getDatabaseName() {
return this.dbName;
}
+ /**
+ * Gets the SerDe parameters; for example see {@link
org.apache.hive.hcatalog.api.HCatCreateTableDesc.Builder#fieldsTerminatedBy(char)}
+ */
+ public Map<String, String> getSerdeParams() {
+ return serdeParams;
+ }
@Override
public String toString() {
@@ -325,7 +339,9 @@ public class HCatCreateTableDesc {
: "outputformat=null")
+ (serde != null ? "serde=" + serde + ", " : "serde=null")
+ (storageHandler != null ? "storageHandler=" + storageHandler
- : "storageHandler=null") + "]";
+ : "storageHandler=null")
+ + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams)
+ + "]";
}
public static class Builder {
@@ -344,6 +360,7 @@ public class HCatCreateTableDesc {
private Map<String, String> tblProps;
private boolean ifNotExists;
private String dbName;
+ private Map<String, String> serdeParams;
private Builder(String dbName, String tableName, List<HCatFieldSchema>
columns) {
@@ -466,7 +483,52 @@ public class HCatCreateTableDesc {
this.fileFormat = format;
return this;
}
-
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder fieldsTerminatedBy(char delimiter) {
+ return serdeParam(serdeConstants.FIELD_DELIM,
Character.toString(delimiter));
+ }
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder escapeChar(char escapeChar) {
+ return serdeParam(serdeConstants.ESCAPE_CHAR,
Character.toString(escapeChar));
+ }
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder collectionItemsTerminatedBy(char delimiter) {
+ return serdeParam(serdeConstants.COLLECTION_DELIM,
Character.toString(delimiter));
+ }
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder mapKeysTerminatedBy(char delimiter) {
+ return serdeParam(serdeConstants.MAPKEY_DELIM,
Character.toString(delimiter));
+ }
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder linesTerminatedBy(char delimiter) {
+ return serdeParam(serdeConstants.LINE_DELIM,
Character.toString(delimiter));
+ }
+ /**
+ * See <i>row_format</i> element of CREATE_TABLE DDL for Hive.
+ */
+ public Builder nullDefinedAs(char nullChar) {
+ return serdeParam(serdeConstants.SERIALIZATION_NULL_FORMAT,
Character.toString(nullChar));
+ }
+ /**
+ * used for setting arbitrary SerDe parameter
+ */
+ public Builder serdeParam(String paramName, String value) {
+ if(serdeParams == null) {
+ serdeParams = new HashMap<String, String>();
+ }
+ serdeParams.put(paramName, value);
+ return this;
+ }
/**
* Builds the HCatCreateTableDesc.
*
@@ -514,6 +576,7 @@ public class HCatCreateTableDesc {
.getName();
LOG.info("Table output format:" + desc.outputformat);
}
+ desc.serdeParams = this.serdeParams;
return desc;
}
}
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java?rev=1564859&r1=1564858&r2=1564859&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
(original)
+++
hive/trunk/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatTable.java
Wed Feb 5 18:18:28 2014
@@ -48,6 +48,7 @@ public class HCatTable {
private String dbName;
private String serde;
private String location;
+ private Map<String, String> serdeParams;
HCatTable(Table hiveTable) throws HCatException {
this.tableName = hiveTable.getTableName();
@@ -73,6 +74,7 @@ public class HCatTable {
tblProps = hiveTable.getParameters();
serde = hiveTable.getSd().getSerdeInfo().getSerializationLib();
location = hiveTable.getSd().getLocation();
+ serdeParams = hiveTable.getSd().getSerdeInfo().getParameters();
}
/**
@@ -200,6 +202,12 @@ public class HCatTable {
public String getLocation() {
return location;
}
+ /**
+ * Returns parameters such as field delimiter,etc.
+ */
+ public Map<String, String> getSerdeParams() {
+ return serdeParams;
+ }
@Override
public String toString() {
@@ -222,6 +230,8 @@ public class HCatTable {
+ ", " : "storageHandler=null")
+ (tblProps != null ? "tblProps=" + tblProps + ", " : "tblProps=null")
+ (serde != null ? "serde=" + serde + ", " : "serde=")
- + (location != null ? "location=" + location : "location=") + "]";
+ + (location != null ? "location=" + location : "location=")
+ + ",serdeParams=" + (serdeParams == null ? "null" : serdeParams)
+ + "]";
}
}
Modified:
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1564859&r1=1564858&r2=1564859&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
(original)
+++
hive/trunk/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
Wed Feb 5 18:18:28 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
+import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
@@ -161,13 +162,28 @@ public class TestHCatClient {
client.dropTable(db, tableOne, true);
HCatCreateTableDesc tableDesc2 = HCatCreateTableDesc.create(db,
- tableTwo, cols).build();
+ tableTwo,
cols).fieldsTerminatedBy('\001').escapeChar('\002').linesTerminatedBy('\003').
+
mapKeysTerminatedBy('\004').collectionItemsTerminatedBy('\005').nullDefinedAs('\006').build();
client.createTable(tableDesc2);
HCatTable table2 = client.getTable(db, tableTwo);
assertTrue(table2.getInputFileFormat().equalsIgnoreCase(
TextInputFormat.class.getName()));
assertTrue(table2.getOutputFileFormat().equalsIgnoreCase(
IgnoreKeyTextOutputFormat.class.getName()));
+ assertTrue("SerdeParams not found", table2.getSerdeParams() != null);
+ assertEquals("checking " + serdeConstants.FIELD_DELIM,
Character.toString('\001'),
+ table2.getSerdeParams().get(serdeConstants.FIELD_DELIM));
+ assertEquals("checking " + serdeConstants.ESCAPE_CHAR,
Character.toString('\002'),
+ table2.getSerdeParams().get(serdeConstants.ESCAPE_CHAR));
+ assertEquals("checking " + serdeConstants.LINE_DELIM,
Character.toString('\003'),
+ table2.getSerdeParams().get(serdeConstants.LINE_DELIM));
+ assertEquals("checking " + serdeConstants.MAPKEY_DELIM,
Character.toString('\004'),
+ table2.getSerdeParams().get(serdeConstants.MAPKEY_DELIM));
+ assertEquals("checking " + serdeConstants.COLLECTION_DELIM,
Character.toString('\005'),
+ table2.getSerdeParams().get(serdeConstants.COLLECTION_DELIM));
+ assertEquals("checking " + serdeConstants.SERIALIZATION_NULL_FORMAT,
Character.toString('\006'),
+ table2.getSerdeParams().get(serdeConstants.SERIALIZATION_NULL_FORMAT));
+
assertEquals((expectedDir + "/" + db + ".db/" + tableTwo).toLowerCase(),
table2.getLocation().toLowerCase());
client.close();
}
