Repository: ambari Updated Branches: refs/heads/trunk 140c9f13d -> 9c6ab0be9
AMBARI-17081 : hive view upload table adding support for date and time format as per hive. (Nitiraj Rathore via dipayanb) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c6ab0be Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c6ab0be Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c6ab0be Branch: refs/heads/trunk Commit: 9c6ab0be9f488f0d89d032c23af0138b729994f8 Parents: 140c9f1 Author: Dipayan Bhowmick <[email protected]> Authored: Thu Jun 9 18:02:08 2016 +0530 Committer: Dipayan Bhowmick <[email protected]> Committed: Thu Jun 9 18:02:54 2016 +0530 ---------------------------------------------------------------------- .../resources/uploads/parsers/ParseUtils.java | 62 ++++++++++++++------ .../hive/resources/uploads/parsers/Parser.java | 4 +- .../hive/resources/upload/ParseUtilsTest.java | 54 +++++++++++++++++ 3 files changed, 98 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9c6ab0be/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java index 3261bfa..e4e2853 100644 --- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/ParseUtils.java @@ -18,23 +18,28 @@ package org.apache.ambari.view.hive.resources.uploads.parsers; +import org.apache.directory.api.util.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; -import static org.apache.ambari.view.hive.client.ColumnDescription.*; +import static org.apache.ambari.view.hive.client.ColumnDescription.DataTypes; public class ParseUtils { protected final static Logger LOG = LoggerFactory.getLogger(ParseUtils.class); - final public static String[] DATE_FORMATS = {"mm/dd/yyyy", "dd/mm/yyyy", "mm-dd-yyyy" /*add more formatss*/}; + final public static DataTypes[] dataTypeList = {DataTypes.BOOLEAN, DataTypes.INT, DataTypes.BIGINT, DataTypes.DOUBLE, DataTypes.CHAR, DataTypes.TIMESTAMP, DataTypes.DATE, DataTypes.STRING}; + private static final String HIVE_DATE_FORMAT = "yyyy-MM-dd"; + + // no strict checking required as it is done by Date parsing + private static final String HIVE_DATE_FORMAT_REGEX = "^[0-9]{4}-[0-9]?[1-9]-[0-9]?[0-9]$"; - final public static DataTypes [] dataTypeList = {DataTypes.BOOLEAN,DataTypes.INT,DataTypes.BIGINT,DataTypes.DOUBLE,DataTypes.CHAR,DataTypes.DATE,DataTypes.STRING}; public static boolean isInteger(Object object) { if (object == null) @@ -59,16 +64,11 @@ public class ParseUtils { return true; String strValue = object.toString(); - if (strValue.equalsIgnoreCase("true") || strValue.equalsIgnoreCase("false")) - return true; - else - return false; + return strValue.equalsIgnoreCase("true") || strValue.equalsIgnoreCase("false"); } public static boolean isString(Object object) { - if (object == null) - return false; - else return true; // any non null can always be interpreted as a string + return object != null; } public static boolean isLong(Object object) { @@ -109,10 +109,8 @@ public class ParseUtils { return true; String str = object.toString().trim(); - if (str.length() == 1) - return true; + return str.length() == 1; - return false; } public static boolean isDate(Object object) { @@ -123,13 +121,36 @@ public class ParseUtils { return true; String str = object.toString(); - for (String format : DATE_FORMATS) { - try { - Date i = new SimpleDateFormat(format).parse(str); - return true; - } catch (Exception e) { + if (Strings.isNotEmpty(str)) { + str = str.trim(); + if (str.matches(HIVE_DATE_FORMAT_REGEX)) { + try { + SimpleDateFormat sdf = new SimpleDateFormat(HIVE_DATE_FORMAT); + sdf.setLenient(false); + Date date = sdf.parse(str); + return true; + } catch (Exception e) { + LOG.debug("error while parsing as date string {}, format {}", str, HIVE_DATE_FORMAT, e); + } } } + return false; + } + + public static boolean isTimeStamp(Object object) { + if (object == null) + return false; + + if (object instanceof Date) + return true; + + String str = object.toString(); + try { + Timestamp ts = Timestamp.valueOf(str); + return true; + } catch (Exception e) { + LOG.debug("error while parsing as timestamp string {}", str, e); + } return false; } @@ -141,6 +162,7 @@ public class ParseUtils { if (isLong(object)) return DataTypes.BIGINT; if (isDouble(object)) return DataTypes.DOUBLE; if (isChar(object)) return DataTypes.CHAR; + if (isTimeStamp(object)) return DataTypes.TIMESTAMP; if (isDate(object)) return DataTypes.DATE; return DataTypes.STRING; @@ -161,6 +183,8 @@ public class ParseUtils { return isChar(object); case DATE: return isDate(object); + case TIMESTAMP: + return isTimeStamp(object); case STRING: return isString(object); @@ -171,7 +195,7 @@ public class ParseUtils { } public static DataTypes detectHiveColumnDataType(List<Object> colValues) { - boolean found = true; + boolean found; for(DataTypes datatype : dataTypeList){ found = true; for(Object object : colValues){ http://git-wip-us.apache.org/repos/asf/ambari/blob/9c6ab0be/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java index 450d896..49f47c7 100644 --- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java +++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/uploads/parsers/Parser.java @@ -88,7 +88,7 @@ public abstract class Parser implements IParser { } int numberOfRows = numberOfPreviewRows; - previewRows = new ArrayList<Row>(numberOfPreviewRows + 1); // size including the header. + previewRows = new ArrayList<>(numberOfPreviewRows + 1); // size including the header. Row headerRow = null; Integer numOfCols = null; @@ -119,8 +119,6 @@ public abstract class Parser implements IParser { for (int colNum = 0; colNum < numOfCols; colNum++) { if(colNum < values.length) { - // detect type - ColumnDescription.DataTypes type = ParseUtils.detectHiveDataType(values[colNum]); newValues[colNum] = values[colNum]; }else{ newValues[colNum] = null; http://git-wip-us.apache.org/repos/asf/ambari/blob/9c6ab0be/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/ParseUtilsTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/ParseUtilsTest.java b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/ParseUtilsTest.java new file mode 100644 index 0000000..b75ed4f --- /dev/null +++ b/contrib/views/hive/src/test/java/org/apache/ambari/view/hive/resources/upload/ParseUtilsTest.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.view.hive.resources.upload; + +import org.apache.ambari.view.hive.resources.uploads.parsers.ParseUtils; +import org.junit.Assert; +import org.junit.Test; + +public class ParseUtilsTest { + @Test + public void testDateFormats() { + Assert.assertTrue(ParseUtils.isDate("1970-01-01")); + Assert.assertTrue(ParseUtils.isDate("1970-01-01 ")); + Assert.assertTrue(ParseUtils.isDate("0001-1-3")); + Assert.assertTrue(ParseUtils.isDate("1996-1-03")); + Assert.assertTrue(ParseUtils.isDate("1996-01-3")); + Assert.assertFalse(ParseUtils.isDate("1970-01-01 01:01:01")); + Assert.assertFalse(ParseUtils.isDate("1970-01-01 23:59:59.999999")); + Assert.assertFalse(ParseUtils.isDate("1970/01/01")); + Assert.assertFalse(ParseUtils.isDate("01-01-1970")); + Assert.assertFalse(ParseUtils.isDate("1970-13-01")); + Assert.assertFalse(ParseUtils.isDate("1970-01-32")); + Assert.assertFalse(ParseUtils.isDate("01/01/1970")); + Assert.assertFalse(ParseUtils.isDate("001-1-3")); + } + + @Test + public void testTimestampFormats() { + Assert.assertFalse(ParseUtils.isTimeStamp("1999-11-30")); + Assert.assertFalse(ParseUtils.isTimeStamp("1999-12-31 23:59")); + Assert.assertTrue(ParseUtils.isTimeStamp("1999-12-31 23:59:59")); + Assert.assertTrue(ParseUtils.isTimeStamp("1999-12-31 23:59:59.100")); + Assert.assertTrue(ParseUtils.isTimeStamp("1999-12-31 23:59:59.999999")); + Assert.assertTrue(ParseUtils.isTimeStamp("1999-12-31 23:59:59.99999999")); + Assert.assertTrue(ParseUtils.isTimeStamp("1999-12-31 23:59:59.999999999")); + Assert.assertFalse(ParseUtils.isTimeStamp("1999-12-31 23:59:59.9999999999")); + Assert.assertFalse(ParseUtils.isTimeStamp("1999/12/31 23:59:59.9999999999")); + } +}
