Repository: carbondata
Updated Branches:
refs/heads/master 71959dad6 -> 425f820bd
[CARBONDATA-1258] CarbonData should not allow loading Date Type values
violating the boundary condition ("0001-01-01" through "9999-12-31")
Spark date type, supporting "0001-01-01" through "9999-12-31". As carbon pushes
the data to spark layer so can support beyond this range.
This closes #1126
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/425f820b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/425f820b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/425f820b
Branch: refs/heads/master
Commit: 425f820bded52998e2da3bc8712a52a1544e7138
Parents: 71959da
Author: mohammadshahidkhan <[email protected]>
Authored: Mon Jul 3 15:48:40 2017 +0530
Committer: manishgupta88 <[email protected]>
Committed: Thu Dec 21 12:22:09 2017 +0530
----------------------------------------------------------------------
.../DateDirectDictionaryGenerator.java | 28 +++++++
.../DateDirectDictionaryGeneratorTest.java | 84 ++++++++++++++++++++
2 files changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index cd32e97..c49af9c 100644
---
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -43,11 +43,33 @@ public class DateDirectDictionaryGenerator implements
DirectDictionaryGenerator
private String dateFormat;
/**
+ * min value supported for date type column
+ */
+ private static final long MIN_VALUE;
+ /**
+ * MAx value supported for date type column
+ */
+ private static final long MAX_VALUE;
+ /**
* Logger instance
*/
private static final LogService LOGGER =
LogServiceFactory.getLogService(DateDirectDictionaryGenerator.class.getName());
+ static {
+ SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
+ df.setTimeZone(TimeZone.getTimeZone("GMT"));
+ long minValue = 0;
+ long maxValue = 0;
+ try {
+ minValue = df.parse("0001-01-01").getTime();
+ maxValue = df.parse("9999-12-31").getTime();
+ } catch (ParseException e) {
+ // the Exception will not occur as constant value is being parsed
+ }
+ MIN_VALUE = minValue;
+ MAX_VALUE = maxValue;
+ }
public DateDirectDictionaryGenerator(String dateFormat) {
this.dateFormat = dateFormat;
initialize();
@@ -142,6 +164,12 @@ public class DateDirectDictionaryGenerator implements
DirectDictionaryGenerator
}
private int generateKey(long timeValue) {
+ if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Value for date type column is not in valid range. Value
considered as null.");
+ }
+ return 1;
+ }
return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git
a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
new file mode 100644
index 0000000..7390556
--- /dev/null
+++
b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.keygenerator.directdictionary;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import
org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test case for the DateDirectDictionaryGenerator
+ */
+public class DateDirectDictionaryGeneratorTest {
+ private DirectDictionaryGenerator dictionaryGenerator = null;
+
+ @Before public void setUp() throws Exception {
+ TimeStampDirectDictionaryGenerator generator = new
TimeStampDirectDictionaryGenerator();
+ dictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(DataTypes.DATE,
+ CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryValueTest() {
+ int surrogateKey =
dictionaryGenerator.generateDirectSurrogateKey("0001-01-01");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void lowerBoundaryInvalidValueTest() {
+ int surrogateKey =
dictionaryGenerator.generateDirectSurrogateKey("0001-01-00");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryValueTest() {
+ int surrogateKey =
dictionaryGenerator.generateDirectSurrogateKey("9999-12-31");
+ Assert.assertTrue(surrogateKey > 1);
+ }
+
+ /**
+ * The generated surrogateKey must be greater than 1
+ *
+ * @throws Exception
+ */
+ @Test public void upperBoundaryInvalidValueTest() {
+ int surrogateKey =
dictionaryGenerator.generateDirectSurrogateKey("10000-12-31");
+ Assert.assertTrue(surrogateKey == 1);
+ }
+
+ @After public void tearDown() throws Exception {
+ dictionaryGenerator = null;
+ }
+}
