This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7492966 Fix OOM (#1074)
7492966 is described below
commit 7492966cdcf6bd7df0b238f51a7b9a509f87761b
Author: Tianan Li <[email protected]>
AuthorDate: Wed Apr 22 20:27:17 2020 +0800
Fix OOM (#1074)
* modify new memtable size estimation and limit the total memoty size of
write log to up to allocate for write size / 10
* update dynamic adapter equation
* modify text size eatimation considering efficiency
* remove useless import
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../db/conf/adapter/IoTDBConfigDynamicAdapter.java | 24 ++++++-----
.../iotdb/db/engine/memtable/AbstractMemTable.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 2 +
.../java/org/apache/iotdb/db/utils/MemUtils.java | 48 +++++++++++-----------
.../db/writelog/node/ExclusiveWriteLogNode.java | 8 ++--
6 files changed, 49 insertions(+), 39 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index d9a406e..c1b3350 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -959,7 +959,7 @@ public class IoTDBConfig {
return walBufferSize;
}
- void setWalBufferSize(int walBufferSize) {
+ public void setWalBufferSize(int walBufferSize) {
this.walBufferSize = walBufferSize;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
index a6a5bb8..ac7d4be 100644
---
a/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
+++
b/server/src/main/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapter.java
@@ -94,6 +94,8 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
*/
private static long CHUNK_METADATA_SIZE_IN_BYTE = 1536L;
+ private static final double WAL_MEMORY_RATIO = 0.1;
+
/**
* Average queue length in memtable pool
*/
@@ -153,6 +155,8 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
if (canAdjust) {
CONFIG.setMaxMemtableNumber(maxMemTableNum);
+ CONFIG.setWalBufferSize(
+ (int) Math.min(Integer.MAX_VALUE, allocateMemoryForWrite *
WAL_MEMORY_RATIO / maxMemTableNum));
CONFIG.setTsFileSizeThreshold(tsFileSizeThreshold);
CONFIG.setMemtableSizeThreshold(memtableSizeInByte);
if (LOGGER.isDebugEnabled() && initialized) {
@@ -181,7 +185,7 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
// when unit is byte, it's likely to cause Long type overflow.
// so when b is larger than Integer.MAC_VALUE use the unit KB.
double a = maxMemTableNum;
- double b = allocateMemoryForWrite - staticMemory;
+ double b = allocateMemoryForWrite * (1 - WAL_MEMORY_RATIO) - staticMemory;
int magnification = b > Integer.MAX_VALUE ? 1024 : 1;
b /= magnification;
double c =
@@ -201,10 +205,9 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
* @return Tsfile byte threshold
*/
private long calcTsFileSizeThreshold(long memTableSize, double ratio) {
- return (long) (
- (allocateMemoryForWrite - maxMemTableNum * memTableSize -
staticMemory) * memTableSize / (
- ratio * maxMemTableNum * CHUNK_METADATA_SIZE_IN_BYTE *
MManager.getInstance()
- .getMaximalSeriesNumberAmongStorageGroups()));
+ return (long) ((allocateMemoryForWrite * (1 - WAL_MEMORY_RATIO) -
maxMemTableNum * memTableSize
+ - staticMemory) * memTableSize / (ratio * maxMemTableNum *
CHUNK_METADATA_SIZE_IN_BYTE
+ * MManager.getInstance().getMaximalSeriesNumberAmongStorageGroups()));
}
/**
@@ -220,17 +223,17 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
@Override
public void addOrDeleteStorageGroup(int diff) throws ConfigAdjusterException
{
totalStorageGroup += diff;
- maxMemTableNum +=
IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() *
diff;
-
- if(!CONFIG.isEnableParameterAdapter()){
- // the maxMemTableNum will also be set in tryToAdaptParameters, this
should not move out
+ maxMemTableNum +=
+
IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() *
diff;
+ if (!CONFIG.isEnableParameterAdapter()) {
CONFIG.setMaxMemtableNumber(maxMemTableNum);
return;
}
if (!tryToAdaptParameters()) {
totalStorageGroup -= diff;
- maxMemTableNum -=
IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() *
diff;
+ maxMemTableNum -=
+
IoTDBDescriptor.getInstance().getConfig().getMemtableNumInEachStorageGroup() *
diff;
throw new ConfigAdjusterException(CREATE_STORAGE_GROUP);
}
}
@@ -297,4 +300,5 @@ public class IoTDBConfigDynamicAdapter implements
IDynamicAdapter {
}
}
+
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index f9de3c7..7297c61 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -93,11 +93,11 @@ public abstract class AbstractMemTable implements IMemTable
{
Object value =
CommonUtils.parseValue(insertPlan.getSchemas()[i].getType(),
insertPlan.getValues()[i]);
+ memSize +=
MemUtils.getRecordSize(insertPlan.getSchemas()[i].getType(), value);
+
write(insertPlan.getDeviceId(), insertPlan.getMeasurements()[i],
insertPlan.getSchemas()[i], insertPlan.getTime(), value);
}
- long recordSizeInByte = MemUtils.getRecordSize(insertPlan);
- memSize += recordSizeInByte;
} catch (QueryProcessException e) {
throw new WriteProcessException(e.getMessage());
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 2ee5c9f..ae81a7d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.ActiveTimeSeriesCounter;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.engine.flush.MemTableFlushTask;
import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
@@ -87,6 +88,7 @@ public class TsFileProcessor {
*/
private volatile boolean shouldClose;
private IMemTable workMemTable;
+
private VersionController versionController;
/**
* this callback is called after the corresponding TsFile is called
endFile().
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
index 5229c48..7946ba6 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/MemUtils.java
@@ -19,8 +19,9 @@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.cache.RamUsageEstimator;
import org.apache.iotdb.db.qp.physical.crud.BatchInsertPlan;
-import org.apache.iotdb.db.qp.physical.crud.InsertPlan;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.write.record.TSRecord;
import org.apache.iotdb.tsfile.write.record.datapoint.BooleanDataPoint;
@@ -43,29 +44,30 @@ public class MemUtils {
}
/**
- * function for getting the record size.
+ * function for getting the value size.
*/
- public static long getRecordSize(InsertPlan insertPlan) {
- long memSize = 0;
- for (int i = 0; i < insertPlan.getValues().length; i++) {
- switch (insertPlan.getSchemas()[i].getType()) {
- case INT32:
- memSize += 8L + 4L; break;
- case INT64:
- memSize += 8L + 8L; break;
- case FLOAT:
- memSize += 8L + 4L; break;
- case DOUBLE:
- memSize += 8L + 8L; break;
- case BOOLEAN:
- memSize += 8L + 1L; break;
- case TEXT:
- memSize += 8L + insertPlan.getValues()[i].length(); break;
- default:
- memSize += 8L + 8L;
- }
+ public static long getRecordSize(TSDataType dataType, Object value) {
+ switch (dataType) {
+ case INT32:
+ return 8L + 4L;
+ case INT64:
+ return 8L + 8L;
+ case FLOAT:
+ return 8L + 4L;
+ case DOUBLE:
+ return 8L + 8L;
+ case BOOLEAN:
+ return 8L + 1L;
+ case TEXT:
+ return 8L + getBinarySize((Binary) value);
+ default:
+ return 8L + 8L;
}
- return memSize;
+ }
+
+ public static long getBinarySize(Binary value) {
+ return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator
+ .sizeOf(value.getValues());
}
public static long getRecordSize(BatchInsertPlan batchInsertPlan, int start,
int end) {
@@ -88,7 +90,7 @@ public class MemUtils {
case TEXT:
memSize += (end - start) * 8L;
for (int j = start; j < end; j++) {
- memSize += ((Binary[])
batchInsertPlan.getColumns()[i])[j].getLength();
+ memSize += getBinarySize(((Binary[])
batchInsertPlan.getColumns()[i])[j]);
}
break;
default:
diff --git
a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index b9b5e0a..dda1a7e 100644
---
a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -46,7 +46,6 @@ public class ExclusiveWriteLogNode implements WriteLogNode,
Comparable<Exclusive
public static final String WAL_FILE_NAME = "wal";
private static final Logger logger =
LoggerFactory.getLogger(ExclusiveWriteLogNode.class);
- private static int logBufferSize =
IoTDBDescriptor.getInstance().getConfig().getWalBufferSize();
private String identifier;
@@ -56,7 +55,8 @@ public class ExclusiveWriteLogNode implements WriteLogNode,
Comparable<Exclusive
private IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- private ByteBuffer logBuffer = ByteBuffer.allocate(logBufferSize);
+ private ByteBuffer logBuffer = ByteBuffer
+ .allocate(IoTDBDescriptor.getInstance().getConfig().getWalBufferSize());
private ReadWriteLock lock = new ReentrantReadWriteLock();
@@ -88,7 +88,9 @@ public class ExclusiveWriteLogNode implements WriteLogNode,
Comparable<Exclusive
sync();
}
} catch (BufferOverflowException e) {
- throw new IOException("Log cannot fit into buffer, please increase
wal_buffer_size", e);
+ throw new IOException(
+ "Log cannot fit into buffer, if you don't enable Dynamic Parameter
Adapter, please increase wal_buffer_size;"
+ + "otherwise, please increase the JVM memory", e);
} finally {
lock.writeLock().unlock();
}
