TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/77fdbfb2 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/77fdbfb2 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/77fdbfb2 Branch: refs/heads/window_function Commit: 77fdbfb2e596c7b1c4d54dff9f4161b9b5378889 Parents: 214301f Author: blrunner <[email protected]> Authored: Tue May 13 00:32:59 2014 +0900 Committer: blrunner <[email protected]> Committed: Tue May 13 00:32:59 2014 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../tajo/catalog/store/HCatalogStore.java | 20 ++++++++++++-------- .../tajo/catalog/store/TestHCatalogStore.java | 14 ++++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index fcd4dbc..75deb0f 100644 --- a/CHANGES +++ b/CHANGES @@ -37,6 +37,8 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-816: NULL delimiter doesn't apply with HCatalogStore. (jaehwa) + TAJO-823: Missing INET4 handling in DatumFactory.cast(). (jihoon) TAJO-803: INSERT INTO without FROM throws ClassCastException. http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 7a694fa..7924af1 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -436,7 +436,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { // If you want to modify table path, you have to modify on Hive cli. if (tableDesc.isExternal()) { table.setTableType(TableType.EXTERNAL_TABLE.name()); - table.getParameters().put("EXTERNAL", "TRUE"); + table.putToParameters("EXTERNAL", "TRUE"); FileSystem fs = tableDesc.getPath().getFileSystem(conf); if (fs.isFile(tableDesc.getPath())) { @@ -479,7 +479,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) { - sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT, + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.CSV)) { @@ -495,14 +495,16 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { // And hive will un-espace this value again. // As a result, user can use right field delimiter. // So, we have to un-escape this value. - sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, + sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, StringEscapeUtils.unescapeJava(fieldDelimiter)); - sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM, + sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter)); + table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER); if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) { - sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT, + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL))); + table.getParameters().remove(StorageConstants.CSVFILE_NULL); } } else if (tableDesc.getMeta().getStoreType().equals(CatalogProtos.StoreType.SEQUENCEFILE)) { String serde = tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_SERDE); @@ -520,17 +522,19 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { // And hive will un-espace this value again. // As a result, user can use right field delimiter. // So, we have to un-escape this value. - sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_FORMAT, + sd.getSerdeInfo().putToParameters(serdeConstants.SERIALIZATION_FORMAT, StringEscapeUtils.unescapeJava(fieldDelimiter)); - sd.getSerdeInfo().getParameters().put(serdeConstants.FIELD_DELIM, + sd.getSerdeInfo().putToParameters(serdeConstants.FIELD_DELIM, StringEscapeUtils.unescapeJava(fieldDelimiter)); + table.getParameters().remove(StorageConstants.SEQUENCEFILE_DELIMITER); } else { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); } if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) { - sd.getSerdeInfo().getParameters().put(serdeConstants.SERIALIZATION_NULL_FORMAT, + table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); + table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); } } else { throw new CatalogException(new NotImplementedException(tableDesc.getMeta().getStoreType().name())); http://git-wip-us.apache.org/repos/asf/tajo/blob/77fdbfb2/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java index a862baf..729184a 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java @@ -174,11 +174,10 @@ public class TestHCatalogStore { @Test public void testTableWithNullValue() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0001")); - options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\\N")); + options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); + options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003")); TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options); - org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); schema.addColumn("s_suppkey", TajoDataTypes.Type.INT4); schema.addColumn("s_name", TajoDataTypes.Type.TEXT); @@ -191,7 +190,6 @@ public class TestHCatalogStore { TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, new Path(warehousePath, new Path(DB_NAME, SUPPLIER))); - store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, SUPPLIER)); @@ -208,7 +206,15 @@ public class TestHCatalogStore { assertEquals(table.getMeta().getOption(StorageConstants.CSVFILE_NULL), table1.getMeta().getOption(StorageConstants.CSVFILE_NULL)); + + assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_DELIMITER), + StringEscapeUtils.escapeJava("\u0002")); + + assertEquals(table1.getMeta().getOption(StorageConstants.CSVFILE_NULL), + StringEscapeUtils.escapeJava("\u0003")); + store.dropTable(DB_NAME, SUPPLIER); + } @Test
