[CARBONDATA-1452] Issue with loading timestamp data beyond cutoff (1)Removed timeValue>=0 condition => this condition will restrict loading proper data when the CARBON_CUTOFF_TIMESTAMP is set before 1970. In this case timeValue will always be < 0 (2) Added test case for the same
This closes #1355 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8791eabf Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8791eabf Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8791eabf Branch: refs/heads/streaming_ingest Commit: 8791eabf0c6f2db385ed9c0886c0aa054a421b2f Parents: 940f4d5 Author: dhatchayani <[email protected]> Authored: Wed Sep 6 15:43:56 2017 +0530 Committer: Ravindra Pesala <[email protected]> Committed: Fri Sep 15 13:54:21 2017 +0530 ---------------------------------------------------------------------- .../TimeStampDirectDictionaryGenerator.java | 13 ++++------ .../TimeStampDirectDictionaryGeneratorTest.java | 26 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java index c8b88d8..e0f5d41 100644 --- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java +++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java @@ -210,15 +210,12 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener } private int generateKey(long timeValue) { - if (timeValue >= 0) { - long time = (timeValue - cutOffTimeStamp) / granularityFactor; - int keyValue = -1; - if (time <= (long) Integer.MAX_VALUE) { - keyValue = (int) time; - } - return keyValue < 0 ? 1 : keyValue + 2; + long time = (timeValue - cutOffTimeStamp) / granularityFactor; + int keyValue = -1; + if (time >= (long) Integer.MIN_VALUE && time <= (long) Integer.MAX_VALUE) { + keyValue = (int) time; } - return 1; + return keyValue < 0 ? 1 : keyValue + 2; } public void initialize() { http://git-wip-us.apache.org/repos/asf/carbondata/blob/8791eabf/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java index 7bb433a..bdc4ca5 100644 --- a/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java +++ b/processing/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGeneratorTest.java @@ -23,6 +23,7 @@ import java.util.TimeZone; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.util.CarbonProperties; +import mockit.Deencapsulation; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -77,6 +78,31 @@ public class TimeStampDirectDictionaryGeneratorTest { * * @throws Exception */ + @Test public void getSurrogateWithCutoff() throws Exception { + SimpleDateFormat timeParser = new SimpleDateFormat(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)); + timeParser.setLenient(false); + TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator(); + long cutOffValue = timeParser.parse("1500-01-01 00:00:00").getTime(); + //setting cutoff time to 1500-01-01 00:00:00 , so we can load data from this time + Deencapsulation.setField(generator, "cutOffTimeStamp", cutOffValue); + int surrogateFromValue = generator.generateDirectSurrogateKey("1500-01-01 00:00:01"); + long valueFromSurrogate = (long) generator.getValueFromSurrogate(surrogateFromValue); + Date date = new Date(valueFromSurrogate / 1000); + Assert.assertEquals("1500-01-01 00:00:01", timeParser.format(date)); + surrogateFromValue = generator.generateDirectSurrogateKey("1499-12-12 00:00:00"); + //1499-12-12 00:00:00 is a value before cut off, so it is a bad record and surrogate should be 1 + Assert.assertEquals(1, surrogateFromValue); + //re setting the value to default + Deencapsulation.setField(generator, "cutOffTimeStamp", 0L); + } + + /** + * The memberString should be retrieved from the actual surrogate key + * + * @throws Exception + */ @Test public void lowerBoundaryValueTest() throws Exception { TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator( ); long valueFromSurrogate = (long) generator.getValueFromSurrogate(2);
