This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new f35b5b7fd9e Add memory usage and series num to log when series
overflow (#10227)
f35b5b7fd9e is described below
commit f35b5b7fd9e2c201959dc7c793945a8623f6ecd3
Author: Marcos_Zyk <[email protected]>
AuthorDate: Tue Jun 20 14:42:18 2023 +0800
Add memory usage and series num to log when series overflow (#10227)
---
.../metadata/SeriesOverflowException.java | 8 ++++---
.../metadata/rescon/ISchemaRegionStatistics.java | 4 ++++
.../metadata/rescon/MemSchemaRegionStatistics.java | 10 +++++++++
.../schemaregion/SchemaRegionMemoryImpl.java | 12 +++++++----
.../schemaregion/SchemaRegionPBTreeImpl.java | 25 +---------------------
5 files changed, 28 insertions(+), 31 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesOverflowException.java
b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesOverflowException.java
index 87e61ab5d6c..5e977fc6572 100644
---
a/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesOverflowException.java
+++
b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesOverflowException.java
@@ -25,10 +25,12 @@ import org.apache.iotdb.rpc.TSStatusCode;
public class SeriesOverflowException extends MetadataException {
- public SeriesOverflowException() {
+ public SeriesOverflowException(long memoryUsage, long seriesNum) {
super(
- "There are too many timeseries in memory, "
- + "please increase MAX_HEAP_SIZE in datanode-env.sh/bat, restart
and create timeseries again.",
+ String.format(
+ "There are too many timeseries in memory. Current memory usage is
%s and series num is %s. "
+ + "Please increase MAX_HEAP_SIZE in datanode-env.sh/bat,
restart and create timeseries again.",
+ memoryUsage, seriesNum),
TSStatusCode.SERIES_OVERFLOW.getStatusCode());
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
index 066ccbed5d1..7b6c28f8ada 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/ISchemaRegionStatistics.java
@@ -39,4 +39,8 @@ public interface ISchemaRegionStatistics {
CachedSchemaRegionStatistics getAsCachedSchemaRegionStatistics();
void clear();
+
+ long getGlobalMemoryUsage();
+
+ long getGlobalSeriesNumber();
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
index 658135bc6c9..be5ededa427 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/rescon/MemSchemaRegionStatistics.java
@@ -151,4 +151,14 @@ public class MemSchemaRegionStatistics implements
ISchemaRegionStatistics {
(templateId, cnt) ->
schemaEngineStatistics.deactivateTemplate(templateId, cnt));
templateUsage.clear();
}
+
+ @Override
+ public long getGlobalMemoryUsage() {
+ return schemaEngineStatistics.getMemoryUsage();
+ }
+
+ @Override
+ public long getGlobalSeriesNumber() {
+ return schemaEngineStatistics.getTotalSeriesNumber();
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
index 26bb62afd68..59f4ee5d1d9 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionMemoryImpl.java
@@ -529,7 +529,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
public void createTimeseries(ICreateTimeSeriesPlan plan, long offset) throws
MetadataException {
if (!regionStatistics.isAllowToCreateNewSeries()) {
- throw new SeriesOverflowException();
+ throw new SeriesOverflowException(
+ regionStatistics.getGlobalMemoryUsage(),
regionStatistics.getGlobalSeriesNumber());
}
IMeasurementMNode<IMemMNode> leafMNode;
@@ -597,7 +598,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
public void createAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan)
throws MetadataException {
int seriesCount = plan.getMeasurements().size();
if (!regionStatistics.isAllowToCreateNewSeries()) {
- throw new SeriesOverflowException();
+ throw new SeriesOverflowException(
+ regionStatistics.getGlobalMemoryUsage(),
regionStatistics.getGlobalSeriesNumber());
}
try {
@@ -769,7 +771,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
@Override
public void createLogicalView(ICreateLogicalViewPlan plan) throws
MetadataException {
if (!regionStatistics.isAllowToCreateNewSeries()) {
- throw new SeriesOverflowException();
+ throw new SeriesOverflowException(
+ regionStatistics.getGlobalMemoryUsage(),
regionStatistics.getGlobalSeriesNumber());
}
try {
@@ -1146,7 +1149,8 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
public void activateSchemaTemplate(IActivateTemplateInClusterPlan plan,
Template template)
throws MetadataException {
if (!regionStatistics.isAllowToCreateNewSeries()) {
- throw new SeriesOverflowException();
+ throw new SeriesOverflowException(
+ regionStatistics.getGlobalMemoryUsage(),
regionStatistics.getGlobalSeriesNumber());
}
try {
diff --git
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionPBTreeImpl.java
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionPBTreeImpl.java
index b9fce764a96..ca4ff08f53a 100644
---
a/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionPBTreeImpl.java
+++
b/server/src/main/java/org/apache/iotdb/db/metadata/schemaregion/SchemaRegionPBTreeImpl.java
@@ -35,7 +35,6 @@ import
org.apache.iotdb.db.exception.metadata.AliasAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import
org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException;
-import org.apache.iotdb.db.exception.metadata.SeriesOverflowException;
import org.apache.iotdb.db.metadata.MetadataConstant;
import org.apache.iotdb.db.metadata.idtable.IDTable;
import org.apache.iotdb.db.metadata.idtable.IDTableManager;
@@ -566,14 +565,6 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
try {
createTimeseries(plan, offset);
done = true;
- } catch (SeriesOverflowException e) {
- logger.warn("Too many timeseries during recovery from MLog, waiting
for PBTree swapping.");
- try {
- Thread.sleep(3000L);
- } catch (InterruptedException e2) {
- logger.error("Exception occurs during timeseries recovery.");
- throw new MetadataException(e2.getMessage());
- }
} catch (AliasAlreadyExistException | PathAlreadyExistException e) {
// skip
done = true;
@@ -664,21 +655,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
}
public void recoverAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan)
throws MetadataException {
- boolean done = false;
- while (!done) {
- try {
- createAlignedTimeSeries(plan);
- done = true;
- } catch (SeriesOverflowException e) {
- logger.warn("Too many timeseries during recovery from MLog, waiting
for PBTree swapping.");
- try {
- Thread.sleep(3000L);
- } catch (InterruptedException e2) {
- logger.error("Exception occurs during timeseries recovery.");
- throw new MetadataException(e2.getMessage());
- }
- }
- }
+ createAlignedTimeSeries(plan);
}
/**
