This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch rc/2.0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/2.0.12 by this push:
new 912dbd424aa Fix subscription cache memory configuration (#18761)
912dbd424aa is described below
commit 912dbd424aa1e310a4058051d63df5219d852d0e
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 12 08:54:16 2026 +0800
Fix subscription cache memory configuration (#18761)
---
.../planner/plan/node/write/InsertTabletNode.java | 5 +-
.../event/cache/SubscriptionPollResponseCache.java | 24 +++++++--
.../node/write/InsertTabletNodeSerdeTest.java | 32 ++++++++++++
.../cache/SubscriptionPollResponseCacheTest.java | 60 ++++++++++++++++++++++
.../apache/iotdb/commons/i18n/CommonMessages.java | 3 ++
.../apache/iotdb/commons/i18n/CommonMessages.java | 3 ++
.../iotdb/commons/conf/CommonDescriptor.java | 16 +++++-
...SubscriptionCacheMemoryUsagePercentageTest.java | 47 +++++++++++++++++
8 files changed, 182 insertions(+), 8 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
index 91c1a6d1c30..fbd273aba7d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java
@@ -1057,7 +1057,10 @@ public class InsertTabletNode extends InsertNode
implements WALEntryValue {
case OBJECT:
Binary[] binaryValues = (Binary[]) column;
for (int j = start; j < end; j++) {
- size += ReadWriteIOUtils.sizeToWrite(binaryValues[j]);
+ // WAL writes a zero length for null values; nullness is preserved
in the bitmap.
+ size +=
+ ReadWriteIOUtils.sizeToWrite(
+ binaryValues[j] == null ? Binary.EMPTY_VALUE :
binaryValues[j]);
}
break;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCache.java
index c4b37b88263..3808722f90e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCache.java
@@ -105,12 +105,14 @@ public class SubscriptionPollResponseCache {
}
private SubscriptionPollResponseCache() {
- final long initMemorySizeInBytes =
-
PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes() / 5;
+ final long totalNonFloatingMemorySizeInBytes =
+
PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes();
+ final float memoryUsagePercentage =
+
SubscriptionConfig.getInstance().getSubscriptionCacheMemoryUsagePercentage();
final long maxMemorySizeInBytes =
- (long)
-
(PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes()
- *
SubscriptionConfig.getInstance().getSubscriptionCacheMemoryUsagePercentage());
+ calculateMaxMemorySizeInBytes(totalNonFloatingMemorySizeInBytes,
memoryUsagePercentage);
+ final long initMemorySizeInBytes =
+ calculateInitialMemorySizeInBytes(totalNonFloatingMemorySizeInBytes,
memoryUsagePercentage);
// properties required by pipe memory control framework
final PipeMemoryBlock allocatedMemoryBlock =
@@ -148,4 +150,16 @@ public class SubscriptionPollResponseCache {
newMemory);
});
}
+
+ static long calculateInitialMemorySizeInBytes(
+ final long totalNonFloatingMemorySizeInBytes, final float
memoryUsagePercentage) {
+ return Math.min(
+ totalNonFloatingMemorySizeInBytes / 5,
+ calculateMaxMemorySizeInBytes(totalNonFloatingMemorySizeInBytes,
memoryUsagePercentage));
+ }
+
+ static long calculateMaxMemorySizeInBytes(
+ final long totalNonFloatingMemorySizeInBytes, final float
memoryUsagePercentage) {
+ return (long) (totalNonFloatingMemorySizeInBytes * memoryUsagePercentage);
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
index a2a407a66d4..4d9fc3df426 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/write/InsertTabletNodeSerdeTest.java
@@ -269,6 +269,38 @@ public class InsertTabletNodeSerdeTest {
new String[] {"\u6e29\u5ea6", "s3", "s4", "s5"},
tmpNode.getMeasurements());
}
+ @Test
+ public void testSerializedSizeWithNullBinaryValue() throws
IllegalPathException, IOException {
+ for (TSDataType dataType :
+ new TSDataType[] {TSDataType.TEXT, TSDataType.STRING, TSDataType.BLOB,
TSDataType.OBJECT}) {
+ InsertTabletNode insertTabletNode = getInsertTabletNodeWithSchema();
+ insertTabletNode.getDataTypes()[0] = dataType;
+ insertTabletNode.getMeasurementSchemas()[0] =
+ new MeasurementSchema(insertTabletNode.getMeasurements()[0],
dataType);
+ Binary value = new Binary(new byte[] {1, 2, 3});
+ insertTabletNode.getColumns()[0] = new Binary[] {value, null,
Binary.EMPTY_VALUE, null};
+ BitMap bitMap = new BitMap(insertTabletNode.getRowCount());
+ bitMap.mark(1);
+ bitMap.mark(3);
+ insertTabletNode.setBitMaps(new BitMap[] {bitMap});
+
+ byte[] bytes = new byte[insertTabletNode.serializedSize()];
+ WALByteBufferForTest walBuffer = new
WALByteBufferForTest(ByteBuffer.wrap(bytes));
+ insertTabletNode.serializeToWAL(walBuffer);
+ Assert.assertFalse(walBuffer.getBuffer().hasRemaining());
+
+ DataInputStream input = new DataInputStream(new
ByteArrayInputStream(bytes));
+ Assert.assertEquals(PlanNodeType.INSERT_TABLET.getNodeType(),
input.readShort());
+ InsertTabletNode restored = InsertTabletNode.deserializeFromWAL(input);
+ Assert.assertArrayEquals(
+ new Binary[] {value, Binary.EMPTY_VALUE, Binary.EMPTY_VALUE,
Binary.EMPTY_VALUE},
+ (Binary[]) restored.getColumns()[0]);
+ Assert.assertTrue(restored.getBitMaps()[0].isMarked(1));
+ Assert.assertTrue(restored.getBitMaps()[0].isMarked(3));
+ Assert.assertFalse(restored.getBitMaps()[0].isMarked(2));
+ }
+ }
+
@Test
public void testSerializeToWALWithoutMeasurementSchemas() throws Exception {
InsertTabletNode insertTabletNode = getInsertTabletNode();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCacheTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCacheTest.java
new file mode 100644
index 00000000000..2afbf610635
--- /dev/null
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/subscription/event/cache/SubscriptionPollResponseCacheTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.subscription.event.cache;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class SubscriptionPollResponseCacheTest {
+
+ private static final long TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES = 1_000;
+
+ @Test
+ public void testInitialMemoryDoesNotExceedConfiguredMaximum() {
+ Assert.assertEquals(
+ 50,
+ SubscriptionPollResponseCache.calculateInitialMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.05F));
+ Assert.assertEquals(
+ 100,
+ SubscriptionPollResponseCache.calculateInitialMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.1F));
+ Assert.assertEquals(
+ 200,
+ SubscriptionPollResponseCache.calculateInitialMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.2F));
+ Assert.assertEquals(
+ 200,
+ SubscriptionPollResponseCache.calculateInitialMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.5F));
+ }
+
+ @Test
+ public void testMaximumMemoryUsesConfiguredPercentage() {
+ Assert.assertEquals(
+ 50,
+ SubscriptionPollResponseCache.calculateMaxMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.05F));
+ Assert.assertEquals(
+ 500,
+ SubscriptionPollResponseCache.calculateMaxMemorySizeInBytes(
+ TOTAL_NON_FLOATING_MEMORY_SIZE_IN_BYTES, 0.5F));
+ }
+}
diff --git
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
index a05c8f9915a..aaafa014cc0 100644
---
a/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/en/org/apache/iotdb/commons/i18n/CommonMessages.java
@@ -315,6 +315,9 @@ public final class CommonMessages {
public static final String
EXCEPTION_DISK_SPACE_WARNING_THRESHOLD_MUST_BE_IN_0_1_BUT_WAS_7B345766 =
"disk_space_warning_threshold must be in [0, 1), but was ";
+ public static final String
+
EXCEPTION_SUBSCRIPTION_CACHE_MEMORY_USAGE_PERCENTAGE_MUST_BE_IN_0_1_BUT_WAS_ARG_57FE2C66
=
+ "subscription_cache_memory_usage_percentage must be in [0, 1], but
was %s.";
public static final String EXCEPTION_FILTER_FUNCTION_WPASS_VALIDATION =
"the value of wpass should be in (0, 1)";
public static final String EXCEPTION_NO_CALCULATE_COLUMNS = "No columns
could be calculated.";
diff --git
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
index c5c6290687e..9f7151bebdc 100644
---
a/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
+++
b/iotdb-core/node-commons/src/main/i18n/zh/org/apache/iotdb/commons/i18n/CommonMessages.java
@@ -216,6 +216,9 @@ public final class CommonMessages {
public static final String
EXCEPTION_THE_ORDER_BY_CLAUSE_OF_THE_DATA_ARGUMENT_MUST_CONTAIN_EXACTLY_THE_TIME_COLUMN_SPECIFIED_BY_THE_TIMECOL_ARGUMENT_4375BAE9
= "DATA 参数的 ORDER BY 子句必须仅包含 TIMECOL 参数指定的时间列。";
public static final String EXCEPTION_UNSUPPORTED_M4_VALUE_TYPE_AF0EF286 =
"不支持的 M4 值类型:";
public static final String
EXCEPTION_DISK_SPACE_WARNING_THRESHOLD_MUST_BE_IN_0_1_BUT_WAS_7B345766 =
"disk_space_warning_threshold 必须在 [0, 1) 范围内,但实际为 ";
+ public static final String
+
EXCEPTION_SUBSCRIPTION_CACHE_MEMORY_USAGE_PERCENTAGE_MUST_BE_IN_0_1_BUT_WAS_ARG_57FE2C66
=
+ "subscription_cache_memory_usage_percentage 必须在 [0, 1] 范围内,但实际为 %s。";
public static final String
LOG_TRUSTED_CHANNEL_FUNCTION_FAILED_INITIATOR_ARG_TARGET_ARG_E4C28443 =
"可信信道功能失效:发起者=%s,目标端=%s";
public static final String
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index cd187a98bed..5b54eb74b4e 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -344,9 +344,9 @@ public class CommonDescriptor {
loadRetryProperties(properties);
}
- private void loadSubscriptionProps(TrimProperties properties) {
+ private void loadSubscriptionProps(TrimProperties properties) throws
IOException {
config.setSubscriptionCacheMemoryUsagePercentage(
- Float.parseFloat(
+ parseSubscriptionCacheMemoryUsagePercentage(
properties.getProperty(
"subscription_cache_memory_usage_percentage",
String.valueOf(config.getSubscriptionCacheMemoryUsagePercentage()))));
@@ -563,6 +563,18 @@ public class CommonDescriptor {
String.valueOf(config.getSubscriptionConsensusIdleSafeTimeBarrierIntervalMs()))));
}
+ static float parseSubscriptionCacheMemoryUsagePercentage(final String value)
throws IOException {
+ final float percentage = Float.parseFloat(value);
+ if (!Float.isFinite(percentage) || percentage < 0 || percentage > 1) {
+ throw new IOException(
+ String.format(
+ CommonMessages
+
.EXCEPTION_SUBSCRIPTION_CACHE_MEMORY_USAGE_PERCENTAGE_MUST_BE_IN_0_1_BUT_WAS_ARG_57FE2C66,
+ percentage));
+ }
+ return percentage;
+ }
+
public void loadRetryProperties(TrimProperties properties) throws
IOException {
config.setRemoteWriteMaxRetryDurationInMs(
Long.parseLong(
diff --git
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonDescriptorSubscriptionCacheMemoryUsagePercentageTest.java
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonDescriptorSubscriptionCacheMemoryUsagePercentageTest.java
new file mode 100644
index 00000000000..1415cd33d88
--- /dev/null
+++
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/conf/CommonDescriptorSubscriptionCacheMemoryUsagePercentageTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.conf;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class CommonDescriptorSubscriptionCacheMemoryUsagePercentageTest {
+
+ @Test
+ public void testValidPercentage() throws IOException {
+ for (final String value : new String[] {"0", "0.05", "0.1", "1"}) {
+ Assert.assertEquals(
+ Float.parseFloat(value),
+ CommonDescriptor.parseSubscriptionCacheMemoryUsagePercentage(value),
+ 0);
+ }
+ }
+
+ @Test
+ public void testInvalidPercentage() {
+ for (final String value : new String[] {"-0.01", "1.01", "NaN",
"Infinity", "-Infinity"}) {
+ Assert.assertThrows(
+ IOException.class,
+ () ->
CommonDescriptor.parseSubscriptionCacheMemoryUsagePercentage(value));
+ }
+ }
+}