Repository: carbondata Updated Branches: refs/heads/master 33b825d7f -> d8bafa34d
[CARBONDATA-2507] enable.offheap.sort not validate in CarbonData In #2274, the value of enable.offheap.sort will transform to false when args[0] not equal to true, including false and other string, like f,any and so on. So we should validate it. This closes #2331 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8bafa34 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8bafa34 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8bafa34 Branch: refs/heads/master Commit: d8bafa34dd4ce31b0596cabe139ea2022ba9d78b Parents: 33b825d Author: xubo245 <xub...@huawei.com> Authored: Wed May 23 16:08:25 2018 +0800 Committer: Jacky Li <jacky.li...@qq.com> Committed: Mon May 28 17:25:16 2018 +0800 ---------------------------------------------------------------------- .../carbondata/core/util/CarbonProperties.java | 17 +++++++++++++ .../carbondata/core/util/SessionParams.java | 2 ++ .../core/CarbonPropertiesValidationTest.java | 25 ++++++++++++++++++++ 3 files changed, 44 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index 391096d..4ee5199 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -56,6 +56,7 @@ import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_ import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT; import static org.apache.carbondata.core.constants.CarbonCommonConstants.CSV_READ_BUFFER_SIZE; import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_AUTO_HANDOFF; +import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_OFFHEAP_SORT; import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT; import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_VECTOR_READER; import static org.apache.carbondata.core.constants.CarbonCommonConstants.HANDOFF_SIZE; @@ -149,6 +150,9 @@ public final class CarbonProperties { case ENABLE_UNSAFE_SORT: validateEnableUnsafeSort(); break; + case ENABLE_OFFHEAP_SORT: + validateEnableOffHeapSort(); + break; case CARBON_CUSTOM_BLOCK_DISTRIBUTION: validateCustomBlockDistribution(); break; @@ -235,6 +239,7 @@ public final class CarbonProperties { validateBlockletGroupSizeInMB(); validateNumberOfColumnPerIORead(); validateEnableUnsafeSort(); + validateEnableOffHeapSort(); validateCustomBlockDistribution(); validateEnableVectorReader(); validateLockType(); @@ -471,6 +476,18 @@ public final class CarbonProperties { } } + private void validateEnableOffHeapSort() { + String value = carbonProperties.getProperty(ENABLE_OFFHEAP_SORT); + boolean isValidBooleanValue = CarbonUtil.validateBoolean(value); + if (!isValidBooleanValue) { + LOGGER.warn("The enable off heap sort value \"" + value + + "\" is invalid. Using the default value \"" + + CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT); + carbonProperties.setProperty(ENABLE_OFFHEAP_SORT, + CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT); + } + } + private void initPropertySet() throws IllegalAccessException { Field[] declaredFields = CarbonCommonConstants.class.getDeclaredFields(); for (Field field : declaredFields) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java index 3823aef..169c003 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java +++ b/core/src/main/java/org/apache/carbondata/core/util/SessionParams.java @@ -33,6 +33,7 @@ import org.apache.carbondata.core.exception.InvalidConfigurationException; import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION; import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_SEARCH_MODE_ENABLE; +import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_OFFHEAP_SORT; import static org.apache.carbondata.core.constants.CarbonCommonConstants.ENABLE_UNSAFE_SORT; import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_ACTION; import static org.apache.carbondata.core.constants.CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE; @@ -145,6 +146,7 @@ public class SessionParams implements Serializable, Cloneable { boolean isValid = false; switch (key) { case ENABLE_UNSAFE_SORT: + case ENABLE_OFFHEAP_SORT: case CARBON_CUSTOM_BLOCK_DISTRIBUTION: case CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE: case CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD: http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8bafa34/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java index bbfe26c..7cc665e 100644 --- a/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java +++ b/core/src/test/java/org/apache/carbondata/core/CarbonPropertiesValidationTest.java @@ -24,6 +24,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.util.CarbonProperties; import junit.framework.TestCase; +import org.junit.Assert; import org.junit.Test; /** @@ -65,6 +66,30 @@ public class CarbonPropertiesValidationTest extends TestCase { CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT.equalsIgnoreCase(valueAfterValidation)); } + @Test public void testValidateEnableOffHeapSort() + throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + Method validateMethodType = + carbonProperties.getClass().getDeclaredMethod("validateEnableOffHeapSort"); + validateMethodType.setAccessible(true); + carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "True"); + assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT) + .equalsIgnoreCase("true")); + carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "false"); + assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT) + .equalsIgnoreCase("false")); + carbonProperties.addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, "xyz"); + assert (carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT) + .equalsIgnoreCase("true")); + String valueBeforeValidation = + carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT); + validateMethodType.invoke(carbonProperties); + String valueAfterValidation = + carbonProperties.getProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT); + assertTrue(valueBeforeValidation.equals(valueAfterValidation)); + assertTrue( + CarbonCommonConstants.ENABLE_OFFHEAP_SORT_DEFAULT.equalsIgnoreCase(valueAfterValidation)); + } + @Test public void testValidateCustomBlockDistribution() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { Method validateMethodType =
