Repository: carbondata Updated Branches: refs/heads/master 2610a6091 -> 27ec6515a
[CARBONDATA-2108]Updated unsafe sort memory configuration Deprecated old property: sort.inmemory.size.inmb Added new property: carbon.sort.storage.inmemory.size.inmb, If user has configured old property then internally it will be converted to new property for ex: If user has configured sort.inmemory.size.inmb then 20% memory will be used as working memory and rest for storage memory This closes #1896 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27ec6515 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27ec6515 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27ec6515 Branch: refs/heads/master Commit: 27ec6515a143dc3b697ac914bfcd4cfe10a49e17 Parents: 2610a60 Author: kumarvishal <kumarvishal.1...@gmail.com> Authored: Wed Jan 31 18:43:02 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Fri Feb 2 23:18:21 2018 +0530 ---------------------------------------------------------------------- .../core/constants/CarbonCommonConstants.java | 5 + .../core/memory/UnsafeMemoryManager.java | 2 +- .../core/memory/UnsafeSortMemoryManager.java | 6 +- .../carbondata/core/util/CarbonProperties.java | 99 ++++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index 87eec8a..8480758 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -1585,6 +1585,11 @@ public final class CarbonCommonConstants { public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT = "true"; + @CarbonProperty + public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB = + "carbon.sort.storage.inmemory.size.inmb"; + public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT = "512"; + private CarbonCommonConstants() { } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java index 4222e14..d3b9b48 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java @@ -47,7 +47,7 @@ public class UnsafeMemoryManager { .getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)); } catch (Exception e) { - size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT); + size = Long.parseLong(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT); LOGGER.info("Wrong memory size given, " + "so setting default value to " + size); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java index c63b320..67bb6cc 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java @@ -75,10 +75,10 @@ public class UnsafeSortMemoryManager { long size; try { size = Long.parseLong(CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB, - CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT)); + .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB, + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT)); } catch (Exception e) { - size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT); + size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT); LOGGER.info("Wrong memory size given, " + "so setting default value to " + size); } if (size < 1024) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index 39a0b80..3dc7b8f 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -223,6 +223,9 @@ public final class CarbonProperties { validateSortIntermediateFilesLimit(); validateEnableAutoHandoff(); validateSchedulerMinRegisteredRatio(); + validateSortMemorySizeInMB(); + validateWorkingMemory(); + validateSortStorageMemory(); } /** @@ -1252,4 +1255,100 @@ public final class CarbonProperties { public void addPropertyToPropertySet(Set<String> externalPropertySet) { propertySet.addAll(externalPropertySet); } + + private void validateSortMemorySizeInMB() { + int sortMemorySizeInMBDefault = + Integer.parseInt(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT); + int sortMemorySizeInMB = 0; + try { + sortMemorySizeInMB = Integer.parseInt( + carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB)); + } catch (NumberFormatException e) { + LOGGER.error( + "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB + + "is Invalid." + " Taking the default value." + + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT); + sortMemorySizeInMB = sortMemorySizeInMBDefault; + } + if (sortMemorySizeInMB < sortMemorySizeInMBDefault) { + LOGGER.error( + "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB + + "is less than default value." + ". Taking the default value." + + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT); + sortMemorySizeInMB = sortMemorySizeInMBDefault; + } + String unsafeWorkingMemoryString = + carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB); + String unsafeSortStorageMemoryString = + carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB); + int workingMemory = 512; + int sortStorageMemory; + if (null == unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString) { + workingMemory = workingMemory > ((sortMemorySizeInMB * 20) / 100) ? + workingMemory : + ((sortMemorySizeInMB * 20) / 100); + sortStorageMemory = sortMemorySizeInMB - workingMemory; + carbonProperties + .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, workingMemory + ""); + carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB, + sortStorageMemory + ""); + } else if (null != unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString) { + carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB, + sortMemorySizeInMB + ""); + } else if (null == unsafeWorkingMemoryString && null != unsafeSortStorageMemoryString) { + carbonProperties + .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, sortMemorySizeInMB + ""); + } + } + + private void validateWorkingMemory() { + int unsafeWorkingMemoryDefault = + Integer.parseInt(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT); + int unsafeWorkingMemory = 0; + try { + unsafeWorkingMemory = Integer.parseInt( + carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB)); + } catch (NumberFormatException e) { + LOGGER.error("The specified value for property " + + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT + "is invalid." + + " Taking the default value." + + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT); + unsafeWorkingMemory = unsafeWorkingMemoryDefault; + } + if (unsafeWorkingMemory < unsafeWorkingMemoryDefault) { + LOGGER.error("The specified value for property " + + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT + + "is less than the default value." + ". Taking the default value." + + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT); + unsafeWorkingMemory = unsafeWorkingMemoryDefault; + } + carbonProperties + .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, unsafeWorkingMemory + ""); + } + + private void validateSortStorageMemory() { + int unsafeSortStorageMemoryDefault = + Integer.parseInt(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT); + int unsafeSortStorageMemory = 0; + try { + unsafeSortStorageMemory = Integer.parseInt(carbonProperties + .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB)); + } catch (NumberFormatException e) { + LOGGER.error("The specified value for property " + + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB + "is invalid." + + " Taking the default value." + + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT); + unsafeSortStorageMemory = unsafeSortStorageMemoryDefault; + } + if (unsafeSortStorageMemory < unsafeSortStorageMemoryDefault) { + LOGGER.error("The specified value for property " + + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB + + "is less than the default value." + " Taking the default value." + + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT); + unsafeSortStorageMemory = unsafeSortStorageMemoryDefault; + } + carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB, + unsafeSortStorageMemory + ""); + } + }
