jt2594838 commented on code in PR #19242: URL: https://github.com/apache/iotdb/pull/19242#discussion_r4560525234
########## integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelSharedClusterIT.java: ########## @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.it.env.MultiEnvFactory; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.ExternalResource; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.Assert.assertEquals; + +/** For pipe tests whose cluster configuration stays constant across all methods in a class. */ +public abstract class AbstractPipeTableModelSharedClusterIT + extends AbstractPipeTableModelDualManualIT { + + private static BaseEnv sharedSender; + private static BaseEnv sharedReceiver; + private static CompletableFuture<Void> startup; + + @Override + protected void setupConfig() { + super.setupConfig(); + // These small functional workloads verify models/patterns, not automatic region expansion. + // CPU-based defaults create many tiny TsFiles and concurrent loads can exhaust the IT heap. + // Keep a region per database; separate databases and the configured replicas remain covered. + for (final BaseEnv env : new BaseEnv[] {senderEnv, receiverEnv}) { + env.getConfig() + .getCommonConfig() + .setDataRegionGroupExtensionPolicy("CUSTOM") + .setDefaultDataRegionGroupNumPerDatabase(1); + } + } + + @ClassRule + public static final ExternalResource CLUSTERS = + new ExternalResource() { + @Override + protected void after() { + // Always release both environments, including when initialization or a test fails. + try { + if (sharedSender != null) { + sharedSender.cleanClusterEnvironment(); + } + } finally { + try { + if (sharedReceiver != null) { + sharedReceiver.cleanClusterEnvironment(); + } + } finally { + sharedSender = null; + sharedReceiver = null; + startup = null; + } + } + } + }; + + @Override + @Before + public void setUp() { + if (sharedSender == null) { + MultiEnvFactory.createEnv(2); + sharedSender = MultiEnvFactory.getEnv(0); + sharedReceiver = MultiEnvFactory.getEnv(1); + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + setupConfig(); + // Join both starts before any test or cleanup runs; retain failures for subsequent methods. + startup = initializeClusters(); + } + senderEnv = sharedSender; + receiverEnv = sharedReceiver; + startup.join(); + } + + protected CompletableFuture<Void> initializeClusters() { + return CompletableFuture.allOf( + CompletableFuture.runAsync(senderEnv::initClusterEnvironment), + CompletableFuture.runAsync(receiverEnv::initClusterEnvironment)); + } + + @Override + protected void cleanEnvironment() { + if (startup == null || startup.isCompletedExceptionally()) { + // A failed start is cleaned by the class rule; do not open connections to unavailable nodes. + return; + } + try { + // Drop all user pipes in both SQL dialects before deleting any database. Otherwise an active + // pipe can recreate the receiver's data during cleanup and contaminate the next method. + dropUserPipes(senderEnv); + dropUserPipes(receiverEnv); + resetCatalog(senderEnv); + resetCatalog(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset shared pipe test clusters", e); + } + } + + private static void dropUserPipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + for (final boolean tableModel : new boolean[] {false, true}) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq() + .setIsTableModel(tableModel) + .setUserName(SessionConfig.DEFAULT_USER)); + assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + final int statusCode = + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(tableModel)) + .getCode(); + // Automatic drop can remove a pipe after the SHOW PIPE snapshot. Review Comment: Treat a user pipe removed by automatic drop after SHOW PIPE as successful cleanup. Ignore only PIPE_NOT_EXIST_ERROR, retain success assertions for every other status, and continue cleaning later pipes and dialects. The real helpers pass 16 controlled RPC checks per version, including unrelated status/transport failures and system-pipe exclusion; both complete Table/Tree AutoDrop classes pass (6 tests). <!-- it-runtime-rationale:126967b6908f457d:1092 --> ########## integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java: ########## @@ -53,20 +59,73 @@ @RunWith(IoTDBTestRunner.class) @Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeDualTreeModelAutoIT { - - @Override - @Before - public void setUp() { - super.setUp(); - } +public class IoTDBPipeAutoDropIT extends AbstractPipeTreeModelSharedClusterIT { @Override protected void setupConfig() { super.setupConfig(); + // Auto-drop waits for completion reports, so collect them frequently in this IT. + senderEnv.getConfig().getCommonConfig().setPipeHeartbeatIntervalSecondsForCollectingPipeMeta(1); senderEnv.getConfig().getConfigNodeConfig().setLeaderDistributionPolicy("HASH"); } + @Override + protected void cleanEnvironment() { + try { + // The unmatched-region scenario creates a table-model pipe while running on the shared + // tree clusters. Drop it before the shared tree cleanup so it cannot recreate the table data. + dropTablePipes(senderEnv); + dropTablePipes(receiverEnv); + super.cleanEnvironment(); + deleteTableDatabases(senderEnv); + deleteTableDatabases(receiverEnv); + } catch (Exception e) { + throw new AssertionError("Failed to reset table state in shared auto-drop clusters", e); + } + } + + private static void dropTablePipes(final BaseEnv env) throws Exception { + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) env.getLeaderConfigNodeConnection()) { + final TShowPipeResp response = + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setUserName(SessionConfig.DEFAULT_USER)); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), response.getStatus().getCode()); + for (final TShowPipeInfo pipe : response.getPipeInfoList()) { + if (!pipe.getId().startsWith(PipeStaticMeta.SYSTEM_PIPE_PREFIX)) { + final int statusCode = + client + .dropPipeExtended(new TDropPipeReq(pipe.getId()).setIsTableModel(true)) + .getCode(); + // Automatic drop can remove a pipe after the SHOW PIPE snapshot. Review Comment: Treat a user pipe removed by automatic drop after SHOW PIPE as successful cleanup. Ignore only PIPE_NOT_EXIST_ERROR, retain success assertions for every other status, and continue cleaning later pipes and dialects. The real helpers pass 16 controlled RPC checks per version, including unrelated status/transport failures and system-pipe exclusion; both complete Table/Tree AutoDrop classes pass (6 tests). <!-- it-runtime-rationale:126967b6908f457d:1093 --> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
