Repository: carbondata Updated Branches: refs/heads/master afe2b669b -> 1bbae2657
[CARBONDATA-3124] Updated log message in UnsafeMemoryManager This closes #2948 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1bbae265 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1bbae265 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1bbae265 Branch: refs/heads/master Commit: 1bbae26571b9dfc2a35ab626bc377924ea3c57a0 Parents: afe2b66 Author: shardul-cr7 <[email protected]> Authored: Fri Nov 23 18:45:44 2018 +0530 Committer: ravipesala <[email protected]> Committed: Wed Nov 28 18:27:53 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/memory/UnsafeMemoryManager.java | 12 ++++-------- docs/faq.md | 8 +++----- 2 files changed, 7 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/1bbae265/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java index 725f003..493ec0b 100644 --- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java +++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java @@ -103,7 +103,7 @@ public class UnsafeMemoryManager { private UnsafeMemoryManager(long totalMemory, MemoryType memoryType) { this.totalMemory = totalMemory; this.memoryType = memoryType; - LOGGER.info("offheap Working Memory manager is created with size " + totalMemory + " with " + LOGGER.info("Offheap Working Memory manager is created with size " + totalMemory + " with " + memoryType); } @@ -128,8 +128,8 @@ public class UnsafeMemoryManager { // not adding on heap memory block to map as JVM will take care of freeing the memory memoryBlock = MemoryAllocator.HEAP.allocate(memoryRequested); if (LOGGER.isDebugEnabled()) { - LOGGER.debug( - String.format("Creating onheap working Memory block (%s) with size:", memoryBlock)); + LOGGER.debug(String + .format("Creating onheap working Memory block (%d) with size: ", memoryBlock.size())); } } return memoryBlock; @@ -143,7 +143,7 @@ public class UnsafeMemoryManager { getMemoryAllocator(memoryBlock.getMemoryType()).free(memoryBlock); memoryUsed -= memoryBlock.size(); memoryUsed = memoryUsed < 0 ? 0 : memoryUsed; - if (LOGGER.isDebugEnabled()) { + if (LOGGER.isDebugEnabled() && memoryBlock.getMemoryType() == MemoryType.OFFHEAP) { LOGGER.debug(String.format("Freeing offheap working memory block (%s) with size: %d, " + "current available memory is: %d", memoryBlock.toString(), memoryBlock.size(), totalMemory - memoryUsed)); @@ -178,10 +178,6 @@ public class UnsafeMemoryManager { taskId, memoryUsed, StringUtils.join(taskIdToOffheapMemoryBlockMap.keySet(), ", "))); } - public synchronized boolean isMemoryAvailable() { - return memoryUsed > totalMemory; - } - public long getUsableMemory() { return totalMemory; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/1bbae265/docs/faq.md ---------------------------------------------------------------------- diff --git a/docs/faq.md b/docs/faq.md index 3ac9a0a..dbcda4f 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -216,20 +216,18 @@ TimeZone.setDefault(TimeZone.getTimeZone("Asia/Shanghai")) ## How to check LRU cache memory footprint? To observe the LRU cache memory footprint in the logs, configure the below properties in log4j.properties file. ``` -log4j.logger.org.apache.carbondata.core.memory.UnsafeMemoryManager = DEBUG log4j.logger.org.apache.carbondata.core.cache.CarbonLRUCache = DEBUG ``` -These properties will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager which will print the information of memory consumed using which the LRU cache size can be decided. **Note:** Enabling the DEBUG log will degrade the query performance. +This property will enable the DEBUG log for the CarbonLRUCache and UnsafeMemoryManager which will print the information of memory consumed using which the LRU cache size can be decided. **Note:** Enabling the DEBUG log will degrade the query performance. Ensure carbon.max.driver.lru.cache.size is configured to observe the current cache size. **Example:** ``` -18/09/26 15:05:28 DEBUG UnsafeMemoryManager: pool-44-thread-1 Memory block (org.apache.carbondata.core.memory.MemoryBlock@21312095) is created with size 10. Total memory used 413Bytes, left 536870499Bytes 18/09/26 15:05:29 DEBUG CarbonLRUCache: main Required size for entry /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge :: 181 Current cache size :: 0 -18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 105available memory: 536870836 -18/09/26 15:05:30 DEBUG UnsafeMemoryManager: main Freeing memory of size: 76available memory: 536870912 18/09/26 15:05:30 INFO CarbonLRUCache: main Removed entry from InMemory lru cache :: /home/target/store/default/stored_as_carbondata_table/Fact/Part0/Segment_0/0_1537954529044.carbonindexmerge ``` +**Note:** If `Removed entry from InMemory LRU cache` are frequently observed in logs, you may have to increase the configured LRU size. +To observe the LRU cache from heap dump, check the heap used by CarbonLRUCache class. ## Getting tablestatus.lock issues When loading data **Symptom**
