This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new dbc4fb74f23 Fix cached nullable table attribute filtering (#19133)
dbc4fb74f23 is described below
commit dbc4fb74f235bfa4cdcd8cd7d8185e24e5f15717
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 17:00:23 2026 +0800
Fix cached nullable table attribute filtering (#19133)
* Fix cached nullable table attribute filtering
* Handle unknown cached attribute predicates
---
.../iotdb/it/env/cluster/config/MppJVMConfig.java | 15 +
.../it/env/cluster/node/AbstractNodeWrapper.java | 1 +
.../it/schema/IoTDBNullableAttributeCacheIT.java | 261 +++++++++
.../schema/TableMetadataPropagationTestBase.java | 450 ++++++++++++++++
.../it/schema/fault/MetadataFaultAgent.java | 600 +++++++++++++++++++++
.../it/schema/fault/MetadataFaultController.java | 244 +++++++++
.../fetcher/DeviceInCacheFilterVisitor.java | 8 +-
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 2 +-
8 files changed, 1578 insertions(+), 3 deletions(-)
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
index d9c51717cf8..7304e3024cd 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppJVMConfig.java
@@ -23,12 +23,26 @@ import org.apache.iotdb.itbase.env.JVMConfig;
import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class MppJVMConfig implements JVMConfig {
private int initHeapSize;
private int maxHeapSize;
private int maxDirectMemorySize;
private String timezone;
+ private List<String> jvmOptions = Collections.emptyList();
+
+ /** Additional arguments for child test-node JVMs, for example a test-only
Java agent. */
+ public void setJvmOptions(List<String> options) {
+ this.jvmOptions = new ArrayList<>(options);
+ }
+
+ public List<String> getJvmOptions() {
+ return Collections.unmodifiableList(jvmOptions);
+ }
@Override
public JVMConfig setInitHeapSize(int initSize) {
@@ -94,6 +108,7 @@ public class MppJVMConfig implements JVMConfig {
this.setMaxHeapSize(config.getMaxHeapSize());
this.setMaxDirectMemorySize(config.getMaxDirectMemorySize());
this.setTimezone(config.getTimezone());
+ this.setJvmOptions(config.getJvmOptions());
}
public static Builder builder() {
diff --git
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
index 8d3f5f2cc9d..a5bc823eafd 100644
---
a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
+++
b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/AbstractNodeWrapper.java
@@ -483,6 +483,7 @@ public abstract class AbstractNodeWrapper implements
BaseNodeWrapper {
File stdoutFile = new File(getLogPath());
List<String> startCmd = new ArrayList<>();
startCmd.add(JAVA_CMD);
+ startCmd.addAll(jvmConfig.getJvmOptions());
if (!SystemUtils.IS_JAVA_1_8) {
startCmd.add("--add-opens=java.base/java.util.concurrent=ALL-UNNAMED");
startCmd.add("--add-opens=java.base/java.lang=ALL-UNNAMED");
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java
new file mode 100644
index 00000000000..f431800bc69
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBNullableAttributeCacheIT.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.relational.it.schema;
+
+import org.apache.iotdb.confignode.manager.lease.ClusterCachePropagator;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import
org.apache.iotdb.relational.it.schema.fault.MetadataFaultController.Node;
+
+import org.awaitility.Awaitility;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(IoTDBTestRunner.class)
+@Category(TableClusterIT.class)
+public class IoTDBNullableAttributeCacheIT extends
TableMetadataPropagationTestBase {
+ /** an unacked node keeps renewing and cannot be treated as fenced. */
+ @Test
+ public void testMissingInvalidationWithLiveLease() throws Exception {
+ invalidationFailure(false);
+ }
+
+ /** Same topology and boundary, with successful execution followed by a lost
transport ACK. */
+ @Test
+ public void testLostInvalidationAckWithLiveLease() throws Exception {
+ invalidationFailure(true);
+ }
+
+ private void invalidationFailure(boolean loseAck) throws Exception {
+ create(loseAck ? "metadata_lost_ack" : "metadata_lost_request", 2);
+ DataNodeWrapper remote = remote();
+ Node target = NODES.get(remote);
+ warm(remote, 2);
+ Map<String, String> lease = target.lease();
+ String hook = loseAck ? ACK : INVALIDATE;
+ target.arm(hook, "DROP", database);
+ for (Node node : NODES.values()) {
+ node.arm(DELETE, "OBSERVE", TABLE);
+ }
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<SQLException> ddl =
+ executor.submit(
+ () -> {
+ try {
+ update("ALTER TABLE sensor DROP COLUMN changing");
+ return null;
+ } catch (SQLException e) {
+ return e;
+ } catch (Exception e) {
+ throw new AssertionError("DDL failed to converge after
restoring its RPC", e);
+ }
+ });
+ try {
+ awaitMatches(target, hook);
+ try (Connection connection = connection(remote);
+ Statement statement = connection.createStatement();
+ ResultSet result =
+ statement.executeQuery(
+ "SELECT kept, reading FROM " + database + ".sensor WHERE
device='d0'")) {
+ TestUtils.assertResultSetEqualUnordered(
+ result, "kept,reading,",
java.util.Collections.singletonList("keep-0,10,"));
+ }
+ long start = System.nanoTime();
+ long fenceMs = Long.parseLong(target.lease().get("threshold"));
+ long proceedMs = fenceMs +
ClusterCachePropagator.DEFAULT_PROCEED_MARGIN_MS;
+ long observationMs =
+ proceedMs
+ + ClusterCachePropagator.BROADCAST_RPC_TIMEOUT_MS
+ * ClusterCachePropagator.BROADCAST_RPC_RETRY
+ + 500;
+ // Cross the coordinator's proceed deadline, while every heartbeat is
still delivered.
+ // This is an invariant checked on each poll, not a sleep used to guess
a DDL phase.
+ Awaitility.await()
+ .pollInterval(100, TimeUnit.MILLISECONDS)
+ .atMost(observationMs + 5_000, TimeUnit.MILLISECONDS)
+ .until(
+ () -> {
+ assertLease(target, lease);
+ if (ddl.isDone()) {
+ assertInvalidationFailure(ddl.get());
+ }
+ for (Node node : NODES.values()) {
+ assertEquals(
+ "Region deletion crossed the invalidation barrier",
+ "0",
+ node.status(DELETE).get("matches"));
+ }
+ long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()
- start);
+ return elapsed > proceedMs + 500 && (ddl.isDone() || elapsed >
observationMs);
+ });
+ assertTrue(Long.parseLong(target.status(hook).get("dropped")) > 1);
+ assertTrue(
+ "The target must continue renewing its live metadata lease",
+ Long.parseLong(target.lease().get("heartbeats"))
+ > Long.parseLong(lease.get("heartbeats")));
+ if (loseAck) {
+ // The exit gate can only be reached after invalidation and release of
the schema lock.
+ assertTrue(target.count(INVALIDATE) > 0);
+ }
+ target.release(hook);
+ SQLException failure = ddl.get(30, TimeUnit.SECONDS);
+ if (failure != null) {
+ assertInvalidationFailure(failure);
+ // DropTableColumnProcedure does not roll back its pre-delete marker.
Reissuing the same
+ // drop is supported: preDeleteColumn adds that marker idempotently
and completes the drop.
+ update("ALTER TABLE sensor DROP COLUMN changing");
+ }
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT device, kept, reading FROM sensor",
+ new String[] {"device", "kept", "reading"},
+ new String[] {"d0,keep-0,10,", "d1,keep-1,11,"},
+ database);
+ update("ALTER TABLE sensor ADD COLUMN changing STRING ATTRIBUTE");
+ assertTwoDevices(null, null, true);
+ update("UPDATE sensor SET changing='after-ddl' WHERE device='d0'");
+ awaitTwoDevices("after-ddl", null, true);
+ assertLease(target, lease);
+ assertAttributeFilter(1, null);
+ } finally {
+ target.release(hook);
+ finish(executor);
+ }
+ }
+
+ /** an acknowledged old detail must not retire a newer clear container. */
+ @Test
+ public void testDegradedAttributeBacklogRecoversWithoutAnotherUpdate()
throws Exception {
+ create("metadata_attribute_degrade", 16);
+ update(
+ "CREATE TABLE retained (device STRING TAG, changing STRING ATTRIBUTE,
reading INT32 FIELD)");
+ update("INSERT INTO retained(time,device,changing,reading)
VALUES(1,'keep','retained',99)");
+ DataNodeWrapper remote = remote();
+ DataNodeWrapper source =
+ EnvFactory.getEnv().getDataNodeWrapperList().stream()
+ .filter(wrapper -> wrapper != remote)
+ .findFirst()
+ .get();
+ Node target = NODES.get(remote);
+ Node sender = NODES.get(source);
+ warm(remote, 16);
+ Map<String, String> lease = target.lease();
+ awaitEmpty(sender);
+ sender.arm(OLD_COMMIT, "HOLD_FIRST", database, "changing=old-ack;");
+ target.arm(CONTAINER, "OBSERVE", database);
+ update("UPDATE sensor SET changing='old-ack' WHERE device='d0'");
+ try {
+ sender.awaitHeld(OLD_COMMIT);
+ Map<String, String> acknowledged = sender.status(OLD_COMMIT);
+ assertEquals(acknowledged.toString(), "detail",
acknowledged.get("batchType"));
+ assertTrue(
+ acknowledged.toString(),
acknowledged.get("attributes").contains("changing=old-ack;"));
+ Map<String, String> baseline = sender.updaters(database);
+ assertEquals(baseline.toString(), "1", baseline.get("found"));
+ assertTrue(baseline.toString(), Integer.parseInt(baseline.get("detail"))
> 0);
+ long capacity = Long.parseLong(baseline.get("capacity"));
+ assertTrue(
+ "This fixture must use a bounded schema budget: " + baseline,
capacity < 1_048_576);
+ assertEquals("true", baseline.get("allowCreate"));
+ long available = capacity - Long.parseLong(baseline.get("usage"));
+ assertTrue(
+ "The fixture needs room for persistent attributes: " + baseline,
available > 32_768);
+ // The store and its uncommitted detail both account for these
different-device values.
+ // Each consumes about 60% of the available budget; dropping the detail
relieves pressure.
+ int length = (int) (available * 3 / 5 / 16);
+ String suffix = String.join("", java.util.Collections.nCopies(length,
"x"));
+ List<String> attributes = new ArrayList<>();
+ for (int i = 0; i < 16; i++) {
+ String attribute = "final-" + i + "-" + suffix;
+ attributes.add(attribute);
+ update("UPDATE sensor SET changing='" + attribute + "' WHERE
device='d" + i + "'");
+ }
+ update("UPDATE sensor SET changing=NULL WHERE device='d0'");
+ attributes.set(0, null);
+ Map<String, String> degraded = sender.updaters(database);
+ assertTrue(
+ degraded.toString(),
+ Long.parseLong(degraded.get("degrades")) >
Long.parseLong(baseline.get("degrades")));
+ assertTrue(degraded.toString(), Integer.parseInt(degraded.get("clear"))
> 0);
+ assertEquals(
+ "The old acknowledged batch is still before commit",
+ "0",
+ sender.status(OLD_COMMIT).get("finished"));
+ target.arm(UPDATE, "HOLD_FIRST", database);
+ sender.release(OLD_COMMIT);
+ target.awaitHeld(UPDATE);
+ assertEquals("clear", target.status(UPDATE).get("batchType"));
+ assertTrue(
+ sender.updaters(database).toString(),
+ Integer.parseInt(sender.updaters(database).get("clear")) > 0);
+ assertLease(target, lease);
+ // From here onwards there are no new UPDATEs until the backlog and
cache have recovered.
+ target.release(UPDATE);
+ awaitFinished(target, UPDATE);
+ Awaitility.await()
+ .atMost(20, TimeUnit.SECONDS)
+ .until(() -> Integer.parseInt(target.status(CONTAINER).get("clear"))
> 0);
+ awaitEmpty(sender);
+ assertDevices(attributes, false);
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT device, changing, reading FROM retained",
+ new String[] {"device", "changing", "reading"},
+ new String[] {"keep,retained,99,"},
+ database);
+ assertLease(target, lease);
+ assertEquals("true", sender.updaters(database).get("allowCreate"));
+ update("UPDATE sensor SET changing='after-pressure' WHERE device='d1'");
+ attributes.set(1, "after-pressure");
+ Awaitility.await()
+ .atMost(20, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertDevices(attributes, false));
+ awaitEmpty(sender);
+ update("INSERT INTO retained(time,device,changing,reading)
VALUES(2,'new','modeled',100)");
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT device, changing, reading FROM retained",
+ new String[] {"device", "changing", "reading"},
+ new String[] {"keep,retained,99,", "new,modeled,100,"},
+ database);
+ assertAttributeFilter(0, null);
+ } finally {
+ sender.release(OLD_COMMIT);
+ if (!target.status(UPDATE).isEmpty()) {
+ target.release(UPDATE);
+ }
+ }
+ }
+}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/TableMetadataPropagationTestBase.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/TableMetadataPropagationTestBase.java
new file mode 100644
index 00000000000..dab287eedcd
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/TableMetadataPropagationTestBase.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.relational.it.schema;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.confignode.i18n.ProcedureMessages;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.it.utils.TestUtils;
+import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+import org.apache.iotdb.relational.it.schema.fault.MetadataFaultController;
+import
org.apache.iotdb.relational.it.schema.fault.MetadataFaultController.Node;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.awaitility.Awaitility;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Real remote attribute pulls, selective invalidation failures and delayed
propagation batches. The
+ * two DataNodes share one Memory cluster; the non-replica is always the cache
under test.
+ */
+@RunWith(IoTDBTestRunner.class)
+@Category(TableClusterIT.class)
+public abstract class TableMetadataPropagationTestBase {
+ protected static final String TABLE = "sensor";
+ protected static final String FILL = "putAttributes:enter";
+ protected static final String UPDATE = "updateAttribute:enter";
+ protected static final String INVALIDATE = "invalidateColumnCache:enter";
+ protected static final String ACK = "invalidateColumnCache:exit";
+ protected static final String DELETE = "deleteColumnData:enter";
+ protected static final String ACCEPT = "handleAttributeUpdate:exit";
+ protected static final String CONTAINER = "handleContainer:enter";
+ protected static final String OLD_COMMIT =
"sendUpdateRequestAndMayShrink:exit";
+ protected static final Map<DataNodeWrapper, Node> NODES = new HashMap<>();
+ protected static MetadataFaultController faults;
+ protected String database;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaEngineMode("Memory")
+
.setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaReplicationFactor(1)
+ .setDataReplicationFactor(1)
+ .setSchemaRegionGroupExtensionPolicy("CUSTOM")
+ .setDataRegionGroupExtensionPolicy("CUSTOM")
+ .setDefaultSchemaRegionGroupNumPerDatabase(1)
+ .setDefaultDataRegionGroupNumPerDatabase(1)
+ .setMetadataLeaseFenceMs(8_000)
+ // At a fixed 512 MiB heap this gives SchemaRegion about 256 KiB.
Cache memory remains
+ // sufficient for exact-device reads, so cache eviction cannot hide
the propagation race.
+ .setSchemaMemoryAllocate("1:199:1");
+ EnvFactory.getEnv().getConfig().getDataNodeJVMConfig().setMaxHeapSize(512);
+ Properties properties = new Properties();
+
properties.setProperty("general_region_attribute_security_service_interval_seconds",
"1");
+
properties.setProperty("general_region_attribute_security_service_timeout_seconds",
"2");
+ properties.setProperty("detail_container_min_degrade_memory_in_bytes",
"8192");
+ ((MppCommonConfig)
EnvFactory.getEnv().getConfig().getDataNodeCommonConfig())
+ .updateProperties(properties);
+ faults =
+ new MetadataFaultController(
+ (MppJVMConfig)
EnvFactory.getEnv().getConfig().getDataNodeJVMConfig());
+ try {
+ EnvFactory.getEnv().initClusterEnvironment(1, 2);
+ for (DataNodeWrapper wrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
+ NODES.put(wrapper, faults.connect(wrapper));
+ }
+ } finally {
+ faults.restoreLaunchEnvironment();
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ try {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ } finally {
+ if (faults != null) {
+ faults.close();
+ }
+ NODES.clear();
+ }
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ faults.reset();
+ if (database != null) {
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("DROP DATABASE " + database);
+ }
+ }
+ }
+
+ protected void create(String name, int devices) throws Exception {
+ database = name;
+ try (Connection connection =
connection(EnvFactory.getEnv().getDataNodeWrapperList().get(0));
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE " + database);
+ statement.execute("USE " + database);
+ statement.execute(
+ "CREATE TABLE sensor (device STRING TAG, changing STRING ATTRIBUTE,
kept STRING ATTRIBUTE, reading INT32 FIELD)");
+ statement.execute(
+ "INSERT INTO sensor(time,device,changing,kept,reading)
VALUES(1,'d0','old-0','keep-0',10)");
+ }
+ DataNodeWrapper remote = remote();
+ DataNodeWrapper owner =
+ EnvFactory.getEnv().getDataNodeWrapperList().stream()
+ .filter(wrapper -> wrapper != remote)
+ .findFirst()
+ .get();
+ // The remaining devices have never been fetched by the remote coordinator
before its gate.
+ try (Connection connection = connection(owner);
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE " + database);
+ for (int i = 1; i < devices; i++) {
+ statement.execute(
+ "INSERT INTO sensor(time,device,changing,kept,reading) VALUES("
+ + (i + 1)
+ + ",'d"
+ + i
+ + "','old-"
+ + i
+ + "','keep-"
+ + i
+ + "',"
+ + (i + 10)
+ + ")");
+ }
+ statement.execute("FLUSH");
+ }
+ }
+
+ protected DataNodeWrapper remote() throws SQLException {
+ DataNodeWrapper replica = null;
+ int regions = 0;
+ try (Connection connection = EnvFactory.getEnv().getTableConnection();
+ Statement statement = connection.createStatement();
+ ResultSet result = statement.executeQuery("SHOW REGIONS")) {
+ while (result.next()) {
+ if (TConsensusGroupType.SchemaRegion.toString()
+ .equals(result.getString(ColumnHeaderConstant.TYPE))
+ &&
database.equals(result.getString(ColumnHeaderConstant.DATABASE))) {
+ regions++;
+ replica =
+ EnvFactory.getEnv()
+
.dataNodeIdToWrapper(result.getInt(ColumnHeaderConstant.DATA_NODE_ID))
+ .get();
+ }
+ }
+ }
+ assertEquals("One sole SchemaRegion replica is required", 1, regions);
+ assertNotNull(replica);
+ for (DataNodeWrapper wrapper :
EnvFactory.getEnv().getDataNodeWrapperList()) {
+ if (wrapper != replica) {
+ return wrapper;
+ }
+ }
+ throw new AssertionError("No remote cache node");
+ }
+
+ protected static Connection connection(DataNodeWrapper wrapper) throws
SQLException {
+ return EnvFactory.getEnv()
+ .getConnection(
+ wrapper,
+ SessionConfig.DEFAULT_USER,
+ SessionConfig.DEFAULT_PASSWORD,
+ BaseEnv.TABLE_SQL_DIALECT);
+ }
+
+ protected void warm(DataNodeWrapper wrapper, int devices) throws Exception {
+ Node node = NODES.get(wrapper);
+ node.arm(FILL, "OBSERVE", database);
+ try (Connection connection = connection(wrapper);
+ Statement statement = connection.createStatement()) {
+ for (int i = 0; i < devices; i++) {
+ try (ResultSet result =
+ statement.executeQuery(
+ "SELECT changing, kept, reading FROM "
+ + database
+ + ".sensor WHERE device='d"
+ + i
+ + "'")) {
+ TestUtils.assertResultSetEqualUnordered(
+ result,
+ "changing,kept,reading,",
+ java.util.Collections.singletonList(
+ "old-" + i + ",keep-" + i + "," + (i + 10) + ","));
+ }
+ }
+ }
+ long fills = Long.parseLong(node.status(FILL).get("matches"));
+ assertTrue("The remote queries must actually fetch their attributes",
fills >= devices - 1);
+ // The same complete keys must now hit the cache whose staleness is under
test.
+ try (Connection connection = connection(wrapper);
+ Statement statement = connection.createStatement()) {
+ for (int i = 0; i < devices; i++) {
+ try (ResultSet result =
+ statement.executeQuery(
+ "SELECT changing FROM " + database + ".sensor WHERE device='d"
+ i + "'")) {
+ assertTrue(result.next());
+ assertEquals("old-" + i, result.getString(1));
+ assertFalse(result.next());
+ }
+ }
+ }
+ assertEquals(Long.toString(fills), node.status(FILL).get("matches"));
+ }
+
+ protected void update(String sql) throws Exception {
+ try (Connection connection =
connection(EnvFactory.getEnv().getDataNodeWrapperList().get(0));
+ Statement statement = connection.createStatement()) {
+ statement.execute("USE " + database);
+ statement.execute(sql);
+ }
+ }
+
+ protected void assertTwoDevices(String first, String second, boolean
readded) {
+ List<String> attributes = new ArrayList<>();
+ attributes.add(first);
+ attributes.add(second);
+ assertDevices(attributes, readded);
+ }
+
+ protected void awaitTwoDevices(String first, String second, boolean readded)
{
+ Awaitility.await()
+ .atMost(20, TimeUnit.SECONDS)
+ .untilAsserted(() -> assertTwoDevices(first, second, readded));
+ }
+
+ protected void assertDevices(List<String> attributes, boolean readded) {
+ List<String> data = new ArrayList<>();
+ List<String> devices = new ArrayList<>();
+ int nulls = 0;
+ for (int i = 0; i < attributes.size(); i++) {
+ String attribute = attributes.get(i);
+ if (attribute == null) {
+ nulls++;
+ }
+ data.add((i + 1) + ",d" + i + "," + attribute + ",keep-" + i + "," + (i
+ 10) + ",");
+ devices.add(
+ "d"
+ + i
+ + ","
+ + (readded ? "keep-" + i + "," + attribute : attribute +
",keep-" + i)
+ + ",");
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT device, changing, kept, reading FROM sensor WHERE device='d"
+ i + "'",
+ new String[] {"device", "changing", "kept", "reading"},
+ new String[] {"d" + i + "," + attribute + ",keep-" + i + "," + (i +
10) + ","},
+ database);
+ }
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT CAST(time AS INT64) AS time, device, changing, kept, reading
FROM sensor",
+ new String[] {"time", "device", "changing", "kept", "reading"},
+ data.toArray(new String[0]),
+ database);
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SHOW DEVICES FROM sensor",
+ readded
+ ? new String[] {"device", "kept", "changing"}
+ : new String[] {"device", "changing", "kept"},
+ devices.toArray(new String[0]),
+ database);
+ TestUtils.tableResultSetEqualTestUnordered(
+ "COUNT DEVICES FROM sensor",
+ new String[] {"count(devices)"},
+ new String[] {attributes.size() + ","},
+ database);
+ TestUtils.tableResultSetEqualTestUnordered(
+ "COUNT DEVICES FROM sensor WHERE changing IS NULL",
+ new String[] {"count(devices)"},
+ new String[] {nulls + ","},
+ database);
+ if (attributes.get(0) != null) {
+ assertAttributeFilter(0, attributes.get(0));
+ }
+ TestUtils.tableResultSetEqualTestUnordered(
+ "COUNT DEVICES FROM sensor WHERE changing='late-old'",
+ new String[] {"count(devices)"},
+ new String[] {"0,"},
+ database);
+ TestUtils.tableResultSetEqualTest(
+ "DESCRIBE sensor",
+ new String[] {"ColumnName", "DataType", "Category"},
+ readded
+ ? new String[] {
+ "time,TIMESTAMP,TIME,",
+ "device,STRING,TAG,",
+ "kept,STRING,ATTRIBUTE,",
+ "reading,INT32,FIELD,",
+ "changing,STRING,ATTRIBUTE,"
+ }
+ : new String[] {
+ "time,TIMESTAMP,TIME,",
+ "device,STRING,TAG,",
+ "changing,STRING,ATTRIBUTE,",
+ "kept,STRING,ATTRIBUTE,",
+ "reading,INT32,FIELD,"
+ },
+ database);
+ }
+
+ protected void assertInvalidationFailure(SQLException failure) {
+ assertNotNull("DDL crossed the live node without an ACK", failure);
+ assertEquals(
+ failure.toString(), TSStatusCode.METADATA_ERROR.getStatusCode(),
failure.getErrorCode());
+ assertTrue(
+ failure.toString(),
+ failure
+ .getMessage()
+ .contains(
+ String.format(
+ ProcedureMessages.INVALIDATE_COLUMN_CACHE_FAILED_FOR_TABLE,
+ "changing",
+ database,
+ TABLE)));
+ }
+
+ protected void assertAttributeFilter(int device, String attribute) {
+ assertAttributePredicate(
+ device, attribute == null ? "changing IS NULL" : "changing='" +
attribute + "'", true);
+ if (attribute == null) {
+ for (String predicate :
+ new String[] {
+ "changing <> 'unmatched'",
+ "changing > 'unmatched'",
+ "changing IN ('unmatched')",
+ "changing NOT IN ('unmatched')",
+ "changing LIKE 'unmatched%'",
+ "changing NOT LIKE 'unmatched%'",
+ "changing <> 'unmatched' OR kept='absent'"
+ }) {
+ assertAttributePredicate(device, predicate, false);
+ }
+ assertAttributePredicate(
+ device, "changing <> 'unmatched' OR kept='keep-" + device + "'",
true);
+ }
+ }
+
+ private void assertAttributePredicate(int device, String attributePredicate,
boolean matches) {
+ String predicate = "device='d" + device + "' AND (" + attributePredicate +
")";
+ TestUtils.tableResultSetEqualTestUnordered(
+ "COUNT DEVICES FROM sensor WHERE " + predicate,
+ new String[] {"count(devices)"},
+ new String[] {matches ? "1," : "0,"},
+ database);
+ // Check nullable data predicates after the complete fault/recovery
sequence. A query failure
+ // must still fail the test, without preventing the delayed batch or later
DDL from being
+ // tested.
+ TestUtils.tableResultSetEqualTestUnordered(
+ "SELECT device, kept, reading FROM sensor WHERE " + predicate,
+ new String[] {"device", "kept", "reading"},
+ matches
+ ? new String[] {"d" + device + ",keep-" + device + "," + (device +
10) + ","}
+ : new String[0],
+ database);
+ }
+
+ protected static void assertLease(Node node, Map<String, String> initial)
throws Exception {
+ Map<String, String> current = node.lease();
+ assertEquals(current.toString(), "false", current.get("fenced"));
+ assertEquals(
+ "Lease recovery must not clear the cache", initial.get("clears"),
current.get("clears"));
+ assertTrue(
+ current.toString(),
+ Long.parseLong(current.get("elapsed")) <
Long.parseLong(current.get("threshold")));
+ assertTrue(
+ current.toString(),
+ Long.parseLong(current.get("heartbeats")) >=
Long.parseLong(initial.get("heartbeats")));
+ }
+
+ protected static void awaitMatches(Node node, String hook) {
+ Awaitility.await()
+ .atMost(20, TimeUnit.SECONDS)
+ .until(() -> Boolean.parseBoolean(node.status(hook).get("captured")));
+ }
+
+ protected static void awaitFinished(Node node, String hook) {
+ Awaitility.await()
+ .atMost(10, TimeUnit.SECONDS)
+ .until(() -> Long.parseLong(node.status(hook).get("completed")) > 0);
+ }
+
+ protected void awaitEmpty(Node sender) {
+ Awaitility.await()
+ .atMost(20, TimeUnit.SECONDS)
+ .untilAsserted(
+ () ->
+ assertEquals(
+ sender.updaters(database).toString(),
+ "0",
+ sender.updaters(database).get("pending")));
+ }
+
+ protected static void finish(ExecutorService executor) throws
InterruptedException {
+ executor.shutdown();
+ if (!executor.awaitTermination(35, TimeUnit.SECONDS)) {
+ executor.shutdownNow();
+ assertTrue("Test executor did not stop", executor.awaitTermination(5,
TimeUnit.SECONDS));
+ }
+ }
+}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultAgent.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultAgent.java
new file mode 100644
index 00000000000..fdcde9766bc
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultAgent.java
@@ -0,0 +1,600 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.relational.it.schema.fault;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.instrument.ClassFileTransformer;
+import java.lang.instrument.Instrumentation;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Launch-time instrumentation for metadata IT child processes only. No
production classes are
+ * replaced on disk. The gates run before the receiver takes its locks, or
after its finally block.
+ * The control channel only arms gates and reads state; it never edits product
caches or versions.
+ */
+public final class MetadataFaultAgent {
+ public static final String INVALIDATE = "invalidateColumnCache";
+ public static final String UPDATE = "updateAttribute";
+ public static final String FILL = "putAttributes";
+ private static final String RPC =
+
"org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl";
+ private static final String FETCHER =
+ "org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/";
+ private static final String UPDATER =
+ "org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/";
+ private static final String LEASE =
"org/apache/iotdb/db/schemaengine/lease/MetadataLeaseManager";
+ private static final Map<String, Rule> RULES = new ConcurrentHashMap<>();
+ private static final Map<String, AtomicLong> COUNTS = new
ConcurrentHashMap<>();
+ private static final Set<Object> UPDATERS = ConcurrentHashMap.newKeySet();
+
+ private MetadataFaultAgent() {}
+
+ public static void premain(String directory, Instrumentation
instrumentation) throws IOException {
+ instrumentation.addTransformer(new Transformer());
+ ServerSocket server = new ServerSocket(0, 16,
InetAddress.getLoopbackAddress());
+ Thread controller = new Thread(() -> serve(server),
"metadata-it-fault-controller");
+ controller.setDaemon(true);
+ controller.start();
+ Path endpoint = Paths.get(directory, ProcessHandle.current().pid() +
".port");
+ Files.write(endpoint,
Collections.singletonList(Integer.toString(server.getLocalPort())));
+ }
+
+ private static void serve(ServerSocket server) {
+ while (!server.isClosed()) {
+ try (Socket socket = server.accept();
+ DataInputStream input = new DataInputStream(socket.getInputStream());
+ DataOutputStream output = new
DataOutputStream(socket.getOutputStream())) {
+ socket.setSoTimeout(5_000);
+ Map<String, String> response;
+ try {
+ String command = input.readUTF();
+ String hook = input.readUTF();
+ String mode = input.readUTF();
+ String match = input.readUTF();
+ String attributes = input.readUTF();
+ response = command(command, hook, mode, match, attributes);
+ } catch (Throwable e) {
+ response = Collections.singletonMap("error", e.toString());
+ }
+ output.writeInt(response.size());
+ for (Map.Entry<String, String> entry : response.entrySet()) {
+ output.writeUTF(entry.getKey());
+ output.writeUTF(entry.getValue());
+ }
+ } catch (IOException e) {
+ // A failed control connection must not disable subsequent
release/cleanup requests.
+ System.err.println("Metadata IT control connection failed: " + e);
+ }
+ }
+ }
+
+ private static Map<String, String> command(
+ String command, String hook, String mode, String match, String
attributes) throws Exception {
+ switch (command) {
+ case "arm":
+ Rule old = RULES.putIfAbsent(hook, new Rule(mode, match, attributes));
+ if (old != null) {
+ throw new IllegalStateException("Gate already armed: " + hook);
+ }
+ return Collections.emptyMap();
+ case "status":
+ Rule rule = RULES.get(hook);
+ return rule == null ? Collections.emptyMap() : rule.snapshot();
+ case "release":
+ Rule toRelease = RULES.get(hook);
+ if (toRelease != null) {
+ toRelease.release.countDown();
+ }
+ return Collections.emptyMap();
+ case "reset":
+ RULES.values().forEach(value -> value.release.countDown());
+ RULES.clear();
+ return Collections.emptyMap();
+ case "lease":
+ Object lease = singleton(LEASE.replace('/', '.'));
+ Map<String, String> state = new HashMap<>();
+ state.put("heartbeats",
Long.toString(count("triggerCheckWithHeartBeat:exit")));
+ state.put("clears",
Long.toString(count("clearCacheAndPullMetaData:enter")));
+ state.put("fenced", invoke(lease, "isFenced").toString());
+ state.put("elapsed", invoke(lease,
"getMillisSinceLastConfigNodeHeartbeat").toString());
+ state.put("threshold", field(lease, "fenceThresholdMs").toString());
+ return state;
+ case "updaters":
+ return updaterState(match);
+ case "count":
+ return Collections.singletonMap("count", Long.toString(count(hook)));
+ default:
+ throw new IllegalArgumentException(command);
+ }
+ }
+
+ private static long count(String hook) {
+ AtomicLong count = COUNTS.get(hook);
+ return count == null ? 0 : count.get();
+ }
+
+ private static Map<String, String> updaterState(String database) throws
Exception {
+ Map<String, String> state = new HashMap<>();
+ int pending = 0;
+ int detail = 0;
+ int clear = 0;
+ int found = 0;
+ for (Object updater : UPDATERS) {
+ if (!field(updater, "databaseName").toString().equals(database)) {
+ continue;
+ }
+ found++;
+ Map<?, ?> containers = (Map<?, ?>) field(updater, "attributeUpdateMap");
+ pending += containers.size();
+ for (Object container : containers.values()) {
+ if
(container.getClass().getSimpleName().equals("UpdateDetailContainer")) {
+ detail++;
+ } else {
+ clear++;
+ }
+ }
+ Object statistics = field(field(updater, "regionStatistics"),
"schemaEngineStatistics");
+ state.put("usage", invoke(statistics, "getMemoryUsage").toString());
+ state.put("capacity", invoke(statistics,
"getMemoryCapacity").toString());
+ state.put("allowCreate", invoke(statistics,
"isAllowToCreateNewSeries").toString());
+ }
+ state.put("found", Integer.toString(found));
+ state.put("pending", Integer.toString(pending));
+ state.put("detail", Integer.toString(detail));
+ state.put("clear", Integer.toString(clear));
+ state.put("degrades", Long.toString(count("degrade:exit")));
+ return state;
+ }
+
+ private static Object singleton(String name) throws Exception {
+ return Class.forName(name).getMethod("getInstance").invoke(null);
+ }
+
+ private static Object invoke(Object object, String method) throws Exception {
+ return object.getClass().getMethod(method).invoke(object);
+ }
+
+ private static Object field(Object object, String name) throws Exception {
+ Class<?> type = object.getClass();
+ while (type != null) {
+ try {
+ Field field = type.getDeclaredField(name);
+ field.setAccessible(true);
+ return field.get(object);
+ } catch (NoSuchFieldException e) {
+ type = type.getSuperclass();
+ }
+ }
+ throw new NoSuchFieldException(name);
+ }
+
+ /** Called only by the instrumented child process, never by the test's
control connection. */
+ public static void hit(String hook, Object receiver, Object[] arguments,
Object result)
+ throws Throwable {
+ COUNTS.computeIfAbsent(hook, key -> new AtomicLong()).incrementAndGet();
+ if
(receiver.getClass().getSimpleName().equals("DeviceAttributeCacheUpdater")) {
+ UPDATERS.add(receiver);
+ }
+ if (hook.endsWith(":exit") && arguments.length > 0) {
+ Rule entryRule = RULES.get(hook.substring(0, hook.length() - 5) +
":enter");
+ if (entryRule != null && arguments[0] == entryRule.firstArgument) {
+ entryRule.completed.incrementAndGet();
+ }
+ }
+ Rule rule = RULES.get(hook);
+ if (rule == null) {
+ return;
+ }
+ String description = Arrays.toString(arguments);
+ if (receiver.getClass().getSimpleName().equals("TableDeviceSchemaCache")
+ && arguments.length > 2
+ && arguments[2] instanceof Map) {
+ StringBuilder values = new StringBuilder();
+ ((Map<?, ?>) arguments[2])
+ .forEach(
+ (key, value) ->
+
values.append(key).append('=').append(binaryString(value)).append(';'));
+ description += " fetchedAttributes=" + values;
+ }
+ if
(receiver.getClass().getSimpleName().equals("GeneralRegionAttributeSecurityService"))
{
+ Map<?, ?> databases = (Map<?, ?>) field(receiver,
"regionId2DatabaseMap");
+ for (Object region : ((Map<?, ?>) arguments[0]).keySet()) {
+ description += " database=" + databases.get(region);
+ }
+ }
+ if (!description.contains(rule.match)) {
+ return;
+ }
+ String attributes = batchAttributes(arguments);
+ if (!attributes.contains(rule.expectedAttributes)) {
+ return;
+ }
+ final long occurrence;
+ // Publish the first batch's identity before a concurrent retry can decide
whether to pass.
+ // Do not retain this monitor while waiting at the gate.
+ synchronized (rule) {
+ occurrence = rule.matches.incrementAndGet();
+ if (occurrence == 1) {
+ rule.description = description.substring(0,
Math.min(description.length(), 4_096));
+ rule.stack = Arrays.toString(Thread.currentThread().getStackTrace());
+ rule.version = version(arguments);
+ rule.batchType = batchType(arguments);
+ rule.attributes = attributes;
+ rule.firstArgument = arguments.length == 0 ? null : arguments[0];
+ rule.captured = true;
+ }
+ }
+ for (Object argument : arguments) {
+ if (argument != null &&
argument.getClass().getSimpleName().equals("UpdateClearContainer")) {
+ rule.clear.incrementAndGet();
+ }
+ }
+ if (rule.release.getCount() == 0) {
+ rule.passed.incrementAndGet();
+ return;
+ }
+ if (rule.mode.equals("HOLD_FIRST") && occurrence == 1) {
+ rule.held.countDown();
+ if (!rule.release.await(120, TimeUnit.SECONDS)) {
+ throw new AssertionError("Metadata IT gate was not released: " + hook);
+ }
+ rule.finished.incrementAndGet();
+ return;
+ }
+ if (rule.mode.equals("DROP")
+ || (rule.mode.equals("HOLD_FIRST")
+ && rule.version != null
+ && rule.version.equals(version(arguments)))) {
+ rule.dropped.incrementAndGet();
+ // Thrift's processor rethrows TTransportException and closes this
connection. In particular,
+ // this does not manufacture an application error status that would
bypass transport retries.
+ throw (Throwable)
+ Class.forName("org.apache.thrift.transport.TTransportException")
+ .getConstructor(String.class)
+ .newInstance("Metadata IT discarded " + hook);
+ }
+ rule.passed.incrementAndGet();
+ }
+
+ private static String version(Object[] arguments) throws Exception {
+ if (arguments.length == 0) {
+ return null;
+ }
+ if (arguments[0] instanceof Map) {
+ StringBuilder versions = new StringBuilder();
+ for (Map.Entry<?, ?> entry : ((Map<?, ?>) arguments[0]).entrySet()) {
+
versions.append(entry.getKey()).append('=').append(invoke(entry.getValue(),
"getLeft"));
+ }
+ return versions.toString();
+ }
+ if
(!arguments[0].getClass().getSimpleName().equals("TAttributeUpdateReq")) {
+ return null;
+ }
+ Map<?, ?> updates = (Map<?, ?>) invoke(arguments[0],
"getAttributeUpdateMap");
+ StringBuilder versions = new StringBuilder();
+ for (Map.Entry<?, ?> entry : updates.entrySet()) {
+
versions.append(entry.getKey()).append('=').append(invoke(entry.getValue(),
"getVersion"));
+ }
+ return versions.toString();
+ }
+
+ private static String binaryString(Object value) {
+ if (value == null) {
+ return "null";
+ }
+ try {
+ for (Method method : value.getClass().getMethods()) {
+ if (method.getName().equals("getStringValue")
+ && method.getParameterCount() == 1
+ &&
method.getParameterTypes()[0].getName().equals("java.nio.charset.Charset")) {
+ return method.invoke(value,
java.nio.charset.StandardCharsets.UTF_8).toString();
+ }
+ }
+ } catch (ReflectiveOperationException ignored) {
+ // Fall back to the value's diagnostic form.
+ }
+ return String.valueOf(value);
+ }
+
+ private static String batchType(Object[] arguments) throws Exception {
+ for (byte[] body : batchBodies(arguments)) {
+ return body[0] == 0 ? "clear" : "detail";
+ }
+ return "empty";
+ }
+
+ private static String batchAttributes(Object[] arguments) throws Exception {
+ List<byte[]> bodies = batchBodies(arguments);
+ if (bodies.isEmpty()) {
+ return "";
+ }
+ Method decode =
+ Class.forName((UPDATER + "DeviceAttributeCacheUpdater").replace('/',
'.'))
+ .getMethod("getContainer", byte[].class);
+ StringBuilder values = new StringBuilder();
+ for (byte[] body : bodies) {
+ // Deserialize a separate diagnostic copy; never alter the request or
product cache.
+ Object container = decode.invoke(null, body);
+ if
(!container.getClass().getSimpleName().equals("UpdateDetailContainer")) {
+ continue;
+ }
+ for (Map.Entry<?, ?> table : ((Map<?, ?>) invoke(container,
"getUpdateMap")).entrySet()) {
+ for (Map.Entry<?, ?> device : ((Map<?, ?>)
table.getValue()).entrySet()) {
+ for (Map.Entry<?, ?> attribute : ((Map<?, ?>)
device.getValue()).entrySet()) {
+ values
+ .append(table.getKey())
+ .append('/')
+ .append(device.getKey())
+ .append(':')
+ .append(attribute.getKey())
+ .append('=')
+ .append(binaryString(attribute.getValue()))
+ .append(';');
+ }
+ }
+ }
+ }
+ return values.substring(0, Math.min(values.length(), 4_096));
+ }
+
+ private static List<byte[]> batchBodies(Object[] arguments) throws Exception
{
+ List<byte[]> bodies = new ArrayList<>();
+ if (arguments.length == 0) {
+ return bodies;
+ }
+ if (arguments[0].getClass().getSimpleName().equals("TAttributeUpdateReq"))
{
+ for (Object update : ((Map<?, ?>) invoke(arguments[0],
"getAttributeUpdateMap")).values()) {
+ bodies.add((byte[]) invoke(update, "getBody"));
+ }
+ } else if (arguments[0] instanceof Map) {
+ // At sendUpdateRequestAndMayShrink's exit this map contains only
acknowledged batches.
+ for (Object pair : ((Map<?, ?>) arguments[0]).values()) {
+ for (Object body : ((Map<?, ?>) invoke(pair, "getRight")).values()) {
+ bodies.add((byte[]) body);
+ }
+ }
+ }
+ return bodies;
+ }
+
+ private static final class Rule {
+ private final String mode;
+ private final String match;
+ private final String expectedAttributes;
+ private final AtomicLong matches = new AtomicLong();
+ private final AtomicLong dropped = new AtomicLong();
+ private final AtomicLong passed = new AtomicLong();
+ private final AtomicLong finished = new AtomicLong();
+ private final AtomicLong clear = new AtomicLong();
+ private final AtomicLong completed = new AtomicLong();
+ private final CountDownLatch held = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+ private volatile String description = "";
+ private volatile String stack = "";
+ private volatile String version;
+ private volatile String batchType = "";
+ private volatile String attributes = "";
+ private volatile Object firstArgument;
+ private volatile boolean captured;
+
+ private Rule(String mode, String match, String expectedAttributes) {
+ this.mode = mode;
+ this.match = match;
+ this.expectedAttributes = expectedAttributes;
+ }
+
+ private Map<String, String> snapshot() {
+ Map<String, String> state = new HashMap<>();
+ state.put("matches", Long.toString(matches.get()));
+ state.put("dropped", Long.toString(dropped.get()));
+ state.put("passed", Long.toString(passed.get()));
+ state.put("finished", Long.toString(finished.get()));
+ state.put("held", Boolean.toString(held.getCount() == 0));
+ state.put("description", description);
+ state.put("stack", stack);
+ state.put("version", String.valueOf(version));
+ state.put("batchType", batchType);
+ state.put("attributes", attributes);
+ state.put("clear", Long.toString(clear.get()));
+ state.put("completed", Long.toString(completed.get()));
+ state.put("captured", Boolean.toString(captured));
+ return state;
+ }
+ }
+
+ private static boolean selected(String owner, String method, boolean exit) {
+ if (owner.equals(RPC)) {
+ return method.equals(INVALIDATE)
+ || method.equals(UPDATE)
+ || method.equals("deleteColumnData");
+ }
+ if (owner.equals(FETCHER + "cache/TableDeviceSchemaCache")) {
+ return method.equals(FILL) && !exit;
+ }
+ if (owner.equals(FETCHER + "TableDeviceCacheAttributeGuard")) {
+ return (method.equals("handleAttributeUpdate") && exit)
+ || (method.equals("handleContainer") && !exit);
+ }
+ if (owner.equals(LEASE)) {
+ return (method.equals("triggerCheckWithHeartBeat") && exit)
+ || (method.equals("clearCacheAndPullMetaData") && !exit);
+ }
+ if (owner.equals(UPDATER + "DeviceAttributeCacheUpdater")) {
+ return exit && (method.equals("afterUpdate") || method.equals("commit"));
+ }
+ if (owner.equals(UPDATER + "GeneralRegionAttributeSecurityService")) {
+ return method.equals("sendUpdateRequestAndMayShrink") && exit;
+ }
+ return owner.equals(UPDATER + "UpdateDetailContainer") &&
method.equals("degrade") && exit;
+ }
+
+ private static final class Transformer implements ClassFileTransformer {
+ @Override
+ public byte[] transform(
+ ClassLoader loader,
+ String name,
+ Class<?> redefining,
+ ProtectionDomain domain,
+ byte[] bytes) {
+ if (!(name.equals(RPC)
+ || name.startsWith(FETCHER)
+ || name.startsWith(UPDATER)
+ || name.equals(LEASE))) {
+ return null;
+ }
+ ClassReader reader = new ClassReader(bytes);
+ // javac can discard this/parameter locals from a return frame when the
original code no
+ // longer reads them. Exit hooks do read them, so retaining the original
frames is invalid.
+ ClassWriter writer = new LoaderClassWriter(reader, loader);
+ reader.accept(new HookVisitor(writer, name), ClassReader.SKIP_FRAMES);
+ return writer.toByteArray();
+ }
+ }
+
+ private static final class LoaderClassWriter extends ClassWriter {
+ private final ClassLoader loader;
+
+ private LoaderClassWriter(ClassReader reader, ClassLoader loader) {
+ super(reader, ClassWriter.COMPUTE_FRAMES);
+ this.loader = loader;
+ }
+
+ @Override
+ protected ClassLoader getClassLoader() {
+ return loader;
+ }
+ }
+
+ private static final class HookVisitor extends ClassVisitor {
+ private final String owner;
+
+ private HookVisitor(ClassVisitor visitor, String owner) {
+ super(Opcodes.ASM9, visitor);
+ this.owner = owner;
+ }
+
+ @Override
+ public MethodVisitor visitMethod(
+ int access, String name, String descriptor, String signature, String[]
exceptions) {
+ MethodVisitor visitor = super.visitMethod(access, name, descriptor,
signature, exceptions);
+ if (!selected(owner, name, false) && !selected(owner, name, true)) {
+ return visitor;
+ }
+ return new HookMethodVisitor(visitor, owner, name, descriptor);
+ }
+ }
+
+ private static final class HookMethodVisitor extends MethodVisitor {
+ private final String owner;
+ private final String method;
+ private final String descriptor;
+
+ private HookMethodVisitor(
+ MethodVisitor visitor, String owner, String method, String descriptor)
{
+ super(Opcodes.ASM9, visitor);
+ this.owner = owner;
+ this.method = method;
+ this.descriptor = descriptor;
+ }
+
+ @Override
+ public void visitCode() {
+ super.visitCode();
+ if (selected(owner, method, false)) {
+ hook(false, false);
+ }
+ }
+
+ @Override
+ public void visitInsn(int opcode) {
+ if ((opcode == Opcodes.RETURN || opcode == Opcodes.ARETURN)
+ && selected(owner, method, true)) {
+ hook(true, opcode == Opcodes.ARETURN);
+ }
+ super.visitInsn(opcode);
+ }
+
+ private void hook(boolean exit, boolean objectReturn) {
+ if (objectReturn) {
+ super.visitInsn(Opcodes.DUP);
+ }
+ super.visitLdcInsn(method + (exit ? ":exit" : ":enter"));
+ if (objectReturn) {
+ super.visitInsn(Opcodes.SWAP);
+ }
+ super.visitVarInsn(Opcodes.ALOAD, 0);
+ if (objectReturn) {
+ super.visitInsn(Opcodes.SWAP);
+ }
+ Type[] arguments = Type.getArgumentTypes(descriptor);
+ super.visitLdcInsn(arguments.length);
+ super.visitTypeInsn(Opcodes.ANEWARRAY, "java/lang/Object");
+ int local = 1;
+ for (int i = 0; i < arguments.length; i++) {
+ super.visitInsn(Opcodes.DUP);
+ super.visitLdcInsn(i);
+ // Selected entry/exit methods have reference parameters only.
+ if (arguments[i].getSort() != Type.OBJECT && arguments[i].getSort() !=
Type.ARRAY) {
+ throw new IllegalStateException("Unexpected hook signature: " +
descriptor);
+ }
+ super.visitVarInsn(Opcodes.ALOAD, local++);
+ super.visitInsn(Opcodes.AASTORE);
+ }
+ if (objectReturn) {
+ super.visitInsn(Opcodes.SWAP);
+ } else {
+ super.visitInsn(Opcodes.ACONST_NULL);
+ }
+ super.visitMethodInsn(
+ Opcodes.INVOKESTATIC,
+ Type.getInternalName(MetadataFaultAgent.class),
+ "hit",
+
"(Ljava/lang/String;Ljava/lang/Object;[Ljava/lang/Object;Ljava/lang/Object;)V",
+ false);
+ }
+ }
+}
diff --git
a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultController.java
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultController.java
new file mode 100644
index 00000000000..db7cd1e0d16
--- /dev/null
+++
b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/fault/MetadataFaultController.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.relational.it.schema.fault;
+
+import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
+import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
+
+import org.awaitility.Awaitility;
+import org.objectweb.asm.ClassReader;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** Owns the agent's temporary jar, launch environment and bounded loopback
control connections. */
+public final class MetadataFaultController implements AutoCloseable {
+ private final Path directory;
+ private final List<Node> nodes = new ArrayList<>();
+ private final MppJVMConfig dataNodeJvmConfig;
+ private final List<String> previousJvmOptions;
+ private boolean launchEnvironmentRestored;
+
+ public MetadataFaultController(MppJVMConfig dataNodeJvmConfig) throws
Exception {
+ this.dataNodeJvmConfig = dataNodeJvmConfig;
+ previousJvmOptions = new ArrayList<>(dataNodeJvmConfig.getJvmOptions());
+ directory =
Files.createTempDirectory(Paths.get("target").toAbsolutePath(),
"metadata-fault-");
+ Path jar = directory.resolve("agent.jar");
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ manifest.getMainAttributes().putValue("Premain-Class",
MetadataFaultAgent.class.getName());
+ try (JarOutputStream output = new
JarOutputStream(Files.newOutputStream(jar), manifest)) {
+ addClass(output, MetadataFaultAgent.class);
+ // Keep ASM inside the temporary agent so child-process classpaths need
no test artifacts.
+ Path asm =
+
Paths.get(ClassReader.class.getProtectionDomain().getCodeSource().getLocation().toURI());
+ try (JarFile source = new JarFile(asm.toFile())) {
+ for (JarEntry entry : source.stream().collect(Collectors.toList())) {
+ if (entry.getName().startsWith("org/objectweb/asm/") &&
!entry.isDirectory()) {
+ output.putNextEntry(new JarEntry(entry.getName()));
+ try (InputStream input = source.getInputStream(entry)) {
+ copy(input, output);
+ }
+ output.closeEntry();
+ }
+ }
+ }
+ }
+ List<String> launchOptions = new ArrayList<>(previousJvmOptions);
+ launchOptions.add("-javaagent:" + jar + "=" + directory);
+ dataNodeJvmConfig.setJvmOptions(launchOptions);
+ }
+
+ /** Called immediately after cluster startup, also on startup failure. */
+ public void restoreLaunchEnvironment() {
+ if (launchEnvironmentRestored) {
+ return;
+ }
+ dataNodeJvmConfig.setJvmOptions(previousJvmOptions);
+ launchEnvironmentRestored = true;
+ }
+
+ public Node connect(DataNodeWrapper wrapper) throws Exception {
+ long pid = wrapper.getInstance().pid();
+ Path endpoint = directory.resolve(pid + ".port");
+ Awaitility.await().atMost(10, TimeUnit.SECONDS).until(() ->
Files.isRegularFile(endpoint));
+ Node node = new
Node(Integer.parseInt(Files.readAllLines(endpoint).get(0)));
+ nodes.add(node);
+ return node;
+ }
+
+ private static void addClass(JarOutputStream output, Class<?> type) throws
IOException {
+ String name = type.getName().replace('.', '/') + ".class";
+ output.putNextEntry(new JarEntry(name));
+ try (InputStream input = type.getClassLoader().getResourceAsStream(name)) {
+ if (input == null) {
+ throw new IOException("Missing agent class: " + name);
+ }
+ copy(input, output);
+ }
+ output.closeEntry();
+ for (Class<?> nested : type.getDeclaredClasses()) {
+ addClass(output, nested);
+ }
+ }
+
+ private static void copy(InputStream input, JarOutputStream output) throws
IOException {
+ byte[] buffer = new byte[8_192];
+ int length;
+ while ((length = input.read(buffer)) != -1) {
+ output.write(buffer, 0, length);
+ }
+ }
+
+ /** Release all in-flight calls before shutting down the cluster. */
+ public void reset() throws IOException {
+ IOException failure = null;
+ for (Node node : nodes) {
+ try {
+ node.reset();
+ } catch (IOException e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+ }
+
+ /** Called after the child processes have stopped. */
+ @Override
+ public void close() throws IOException {
+ restoreLaunchEnvironment();
+ try (Stream<Path> paths = Files.walk(directory)) {
+ for (Path path :
paths.sorted(Comparator.reverseOrder()).collect(Collectors.toList())) {
+ Files.delete(path);
+ }
+ }
+ }
+
+ public static final class Node {
+ private final int port;
+
+ private Node(int port) {
+ this.port = port;
+ }
+
+ public void arm(String hook, String mode, String database) throws
IOException {
+ arm(hook, mode, database, "");
+ }
+
+ public void arm(String hook, String mode, String database, String
attributes)
+ throws IOException {
+ request("arm", hook, mode, database, attributes);
+ }
+
+ public Map<String, String> status(String hook) throws IOException {
+ return request("status", hook, "", "");
+ }
+
+ public void release(String hook) throws IOException {
+ request("release", hook, "", "");
+ }
+
+ public void reset() throws IOException {
+ request("reset", "", "", "");
+ }
+
+ public Map<String, String> lease() throws IOException {
+ return request("lease", "", "", "");
+ }
+
+ public Map<String, String> updaters(String database) throws IOException {
+ return request("updaters", "", "", database);
+ }
+
+ public long count(String hook) throws IOException {
+ return Long.parseLong(request("count", hook, "", "").get("count"));
+ }
+
+ public void awaitHeld(String hook) {
+ Awaitility.await()
+ .atMost(30, TimeUnit.SECONDS)
+ .untilAsserted(
+ () -> {
+ Map<String, String> state = status(hook);
+ if (!Boolean.parseBoolean(state.get("held"))) {
+ throw new AssertionError(hook + " did not reach the gate: "
+ state);
+ }
+ });
+ }
+
+ private Map<String, String> request(String command, String hook, String
mode, String match)
+ throws IOException {
+ return request(command, hook, mode, match, "");
+ }
+
+ private Map<String, String> request(
+ String command, String hook, String mode, String match, String
attributes)
+ throws IOException {
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(InetAddress.getLoopbackAddress(),
port), 5_000);
+ socket.setSoTimeout(5_000);
+ try (DataOutputStream output = new
DataOutputStream(socket.getOutputStream());
+ DataInputStream input = new
DataInputStream(socket.getInputStream())) {
+ output.writeUTF(command);
+ output.writeUTF(hook);
+ output.writeUTF(mode);
+ output.writeUTF(match);
+ output.writeUTF(attributes);
+ output.flush();
+ Map<String, String> result = new HashMap<>();
+ int size = input.readInt();
+ for (int i = 0; i < size; i++) {
+ result.put(input.readUTF(), input.readUTF());
+ }
+ if (result.containsKey("error")) {
+ throw new IOException("Metadata IT controller: " +
result.get("error"));
+ }
+ return result;
+ }
+ }
+ }
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java
index fe56024751d..e62f76be0df 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/DeviceInCacheFilterVisitor.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import
org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry;
import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.utils.Binary;
import java.util.HashMap;
import java.util.List;
@@ -64,11 +65,14 @@ public class DeviceInCacheFilterVisitor extends
SchemaFilterVisitor<AlignedDevic
@Override
public Boolean visitAttributeFilter(
final AttributeFilter filter, final AlignedDeviceEntry deviceEntry) {
+ final Binary attributeValue =
+
deviceEntry.getAttributeColumnValues()[attributeIndexMap.get(filter.getKey())];
return filter
.getChild()
.accept(
StringValueFilterVisitor.getInstance(),
-
deviceEntry.getAttributeColumnValues()[attributeIndexMap.get(filter.getKey())]
- .getStringValue(TSFileConfig.STRING_CHARSET));
+ attributeValue == null
+ ? null
+ : attributeValue.getStringValue(TSFileConfig.STRING_CHARSET));
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
index 7a928acbec8..04b3807699a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaFetcher.java
@@ -415,7 +415,7 @@ public class TableDeviceSchemaFetcher {
// will be a memory source and combine filter
check =
Objects.nonNull(fuzzyFilter) &&
!TreeViewSchema.isTreeViewTable(tableInstance)
- ? o -> filterVisitor.process(fuzzyFilter, o)
+ ? o -> Boolean.TRUE.equals(filterVisitor.process(fuzzyFilter,
o))
: null;
}