Github user kumarvishal09 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/1185#discussion_r128228585
--- Diff:
core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
---
@@ -75,76 +79,67 @@
private MemoryAllocator allocator;
- private long minimumMemory;
-
- // for debug purpose
- private Set<MemoryBlock> set = new HashSet<>();
-
private UnsafeMemoryManager(long totalMemory, MemoryAllocator allocator)
{
this.totalMemory = totalMemory;
this.allocator = allocator;
- long numberOfCores = CarbonProperties.getInstance().getNumberOfCores();
- long sortMemoryChunkSize =
CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
- sortMemoryChunkSize = sortMemoryChunkSize * 1024 * 1024;
- long totalWorkingMemoryForAllThreads = sortMemoryChunkSize *
numberOfCores;
- if (totalWorkingMemoryForAllThreads >= totalMemory) {
- throw new RuntimeException("Working memory should be less than total
memory configured, "
- + "so either reduce the loading threads or increase the memory
size. "
- + "(Number of threads * number of threads) should be less than
total unsafe memory");
- }
- minimumMemory = totalWorkingMemoryForAllThreads;
- LOGGER.info("Memory manager is created with size " + totalMemory + "
with " + allocator
- + " and minimum reserve memory " + minimumMemory);
+ LOGGER
+ .info("Working Memory manager is created with size " + totalMemory
+ " with " + allocator);
}
- private synchronized MemoryBlock allocateMemory(long memoryRequested) {
+ private synchronized MemoryBlock allocateMemory(long taskId, long
memoryRequested) {
if (memoryUsed + memoryRequested <= totalMemory) {
MemoryBlock allocate = allocator.allocate(memoryRequested);
memoryUsed += allocate.size();
- if (LOGGER.isDebugEnabled()) {
- set.add(allocate);
- LOGGER.error("Memory block (" + allocate + ") is created with size
" + allocate.size() +
- ". Total memory used " + memoryUsed + "Bytes, left " +
getAvailableMemory() + "Bytes");
+ Set<MemoryBlock> listOfMemoryBlock =
taskIdToMemoryBlockMap.get(taskId);
+ if (null == listOfMemoryBlock) {
+ listOfMemoryBlock = new HashSet<>();
+ taskIdToMemoryBlockMap.put(taskId, listOfMemoryBlock);
}
+ listOfMemoryBlock.add(allocate);
return allocate;
}
return null;
}
- public synchronized void freeMemory(MemoryBlock memoryBlock) {
+ public synchronized void freeMemory(long taskId,MemoryBlock memoryBlock)
{
+ taskIdToMemoryBlockMap.get(taskId).remove(memoryBlock);
--- End diff --
ok
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
