This is an automated email from the ASF dual-hosted git repository.
iamhucong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 534e7f8eab7 Optimize dml reset data logic and add cache for data init
to reduce e2e execute time (#38867)
534e7f8eab7 is described below
commit 534e7f8eab7e6bfcd3159928cca4386440948c55
Author: Zhengqiang Duan <[email protected]>
AuthorDate: Mon Jun 22 10:58:15 2026 +0800
Optimize dml reset data logic and add cache for data init to reduce e2e
execute time (#38867)
* Optimize dml reset data logic and add cache for data init to reduce e2e
execute time
* Optimize SQL E2E execution performance
* Fix SQL E2E DDL index wait
* Fix SQL E2E DDL table wait
* Refine SQL E2E collection declarations
* Refactor DataSetResetScopeCalculator and DQLExclusiveExecutionDetector
---
test/e2e/sql/pom.xml | 10 +
.../e2e/sql/cases/SQLE2ETestCasesXMLLoader.java | 4 +-
.../test/e2e/sql/cases/dataset/DataSetLoader.java | 31 +-
.../e2e/sql/env/DataSetEnvironmentManager.java | 272 +++++++++---
.../e2e/sql/env/DataSetResetScopeCalculator.java | 479 +++++++++++++++++++++
.../test/e2e/sql/env/SQLE2EEnvironmentEngine.java | 6 +-
.../compose/ContainerComposerRegistry.java | 26 +-
.../test/e2e/sql/it/sql/ddl/DDLE2EIT.java | 150 ++++++-
.../test/e2e/sql/it/sql/dml/BaseDMLE2EIT.java | 14 +-
.../test/e2e/sql/it/sql/dml/BatchDMLE2EIT.java | 3 -
.../e2e/sql/it/sql/dql/AdditionalDQLE2EIT.java | 62 ++-
.../test/e2e/sql/it/sql/dql/BaseDQLE2EIT.java | 54 ++-
.../it/sql/dql/DQLExclusiveExecutionDetector.java | 143 ++++++
.../sql/dql/DQLExclusiveExecutionDetectorTest.java | 75 ++++
.../test/e2e/sql/it/sql/dql/GeneralDQLE2EIT.java | 16 +-
.../src/test/resources/junit-platform.properties | 22 +
16 files changed, 1233 insertions(+), 134 deletions(-)
diff --git a/test/e2e/sql/pom.xml b/test/e2e/sql/pom.xml
index a01808fa94c..6735ad984a4 100644
--- a/test/e2e/sql/pom.xml
+++ b/test/e2e/sql/pom.xml
@@ -33,6 +33,16 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.shardingsphere</groupId>
+
<artifactId>shardingsphere-database-connector-postgresql</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.shardingsphere</groupId>
+
<artifactId>shardingsphere-database-connector-opengauss</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.postgresql</groupId>
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/SQLE2ETestCasesXMLLoader.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/SQLE2ETestCasesXMLLoader.java
index 1e4f5c8296f..17119f8b1bf 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/SQLE2ETestCasesXMLLoader.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/SQLE2ETestCasesXMLLoader.java
@@ -37,9 +37,9 @@ import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collection;
import java.util.Collections;
-import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
@@ -51,7 +51,7 @@ public final class SQLE2ETestCasesXMLLoader {
private static final SQLE2ETestCasesXMLLoader INSTANCE = new
SQLE2ETestCasesXMLLoader();
- private final Map<SQLCommandType, Collection<SQLE2ETestCaseContext>>
testCaseContexts = new LinkedHashMap<>();
+ private final Map<SQLCommandType, Collection<SQLE2ETestCaseContext>>
testCaseContexts = new ConcurrentHashMap<>();
/**
* Get singleton instance.
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/dataset/DataSetLoader.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/dataset/DataSetLoader.java
index 5e9e7e33f17..80f8dcba29f 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/dataset/DataSetLoader.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/cases/dataset/DataSetLoader.java
@@ -29,6 +29,8 @@ import javax.xml.bind.JAXBException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Data set loader.
@@ -38,6 +40,10 @@ public final class DataSetLoader {
private static final String DATA_SET_FOLDER_NAME = "dataset";
+ private static final JAXBContext JAXB_CONTEXT = createJAXBContext();
+
+ private static final Map<String, DataSet> DATA_SET_CACHE = new
ConcurrentHashMap<>();
+
/**
* Load data set.
*
@@ -48,10 +54,29 @@ public final class DataSetLoader {
* @param dataSetFile name of data set file
* @return data set
*/
- @SneakyThrows({JAXBException.class, IOException.class})
public static DataSet load(final String parentPath, final String scenario,
final DatabaseType databaseType, final Mode mode, final String dataSetFile) {
- try (FileReader reader = new FileReader(getFile(parentPath, scenario,
databaseType, mode, dataSetFile))) {
- return (DataSet)
JAXBContext.newInstance(DataSet.class).createUnmarshaller().unmarshal(reader);
+ return load(getFile(parentPath, scenario, databaseType, mode,
dataSetFile));
+ }
+
+ /**
+ * Load data set.
+ *
+ * @param dataSetFile data set file
+ * @return data set
+ */
+ public static DataSet load(final String dataSetFile) {
+ return DATA_SET_CACHE.computeIfAbsent(dataSetFile,
DataSetLoader::loadDataSet);
+ }
+
+ @SneakyThrows(JAXBException.class)
+ private static JAXBContext createJAXBContext() {
+ return JAXBContext.newInstance(DataSet.class);
+ }
+
+ @SneakyThrows({JAXBException.class, IOException.class})
+ private static DataSet loadDataSet(final String dataSetFile) {
+ try (FileReader reader = new FileReader(dataSetFile)) {
+ return (DataSet)
JAXB_CONTEXT.createUnmarshaller().unmarshal(reader);
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetEnvironmentManager.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetEnvironmentManager.java
index bfeab25433b..e95667e3593 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetEnvironmentManager.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetEnvironmentManager.java
@@ -30,6 +30,7 @@ import
org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine;
import
org.apache.shardingsphere.infra.executor.kernel.thread.ExecutorServiceManager;
import
org.apache.shardingsphere.infra.expr.entry.InlineExpressionParserFactory;
import org.apache.shardingsphere.test.e2e.sql.cases.dataset.DataSet;
+import org.apache.shardingsphere.test.e2e.sql.cases.dataset.DataSetLoader;
import
org.apache.shardingsphere.test.e2e.sql.cases.dataset.metadata.DataSetColumn;
import
org.apache.shardingsphere.test.e2e.sql.cases.dataset.metadata.DataSetMetaData;
import org.apache.shardingsphere.test.e2e.sql.cases.dataset.row.DataSetRow;
@@ -37,23 +38,27 @@ import
org.apache.shardingsphere.test.e2e.sql.cases.value.SQLValue;
import org.apache.shardingsphere.test.e2e.sql.cases.value.SQLValueGroup;
import javax.sql.DataSource;
-import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
-import java.io.FileReader;
import java.io.IOException;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeSet;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.stream.Collectors;
/**
* Data set environment manager.
@@ -65,6 +70,16 @@ public final class DataSetEnvironmentManager {
private static final String DATA_COLUMN_DELIMITER = ", ";
+ private static final Map<String, ResetPlan> RESET_PLAN_CACHE = new
ConcurrentHashMap<>();
+
+ private static final Map<String, DatabaseType>
DATA_SOURCE_DATABASE_TYPE_CACHE = new ConcurrentHashMap<>();
+
+ private static final Map<String, String> INSERT_SQL_CACHE = new
ConcurrentHashMap<>();
+
+ private static final Map<String, String> TRUNCATE_SQL_CACHE = new
ConcurrentHashMap<>();
+
+ private final String dataSetFile;
+
private final DataSet dataSet;
private final Map<String, DataSource> dataSourceMap;
@@ -72,54 +87,69 @@ public final class DataSetEnvironmentManager {
private final DatabaseType databaseType;
public DataSetEnvironmentManager(final String dataSetFile, final
Map<String, DataSource> dataSourceMap, final DatabaseType databaseType) throws
IOException, JAXBException {
- try (FileReader reader = new FileReader(dataSetFile)) {
- dataSet = (DataSet)
JAXBContext.newInstance(DataSet.class).createUnmarshaller().unmarshal(reader);
- }
+ this.dataSetFile = dataSetFile;
+ dataSet = DataSetLoader.load(dataSetFile);
this.dataSourceMap = dataSourceMap;
this.databaseType = databaseType;
}
/**
- * Fill data.
+ * Fill data for specified tables, or all tables when table names are
empty.
+ *
+ * @param tableNames table names
*/
@SneakyThrows({SQLException.class, InterruptedException.class,
ExecutionException.class})
- public void fillData() {
- Map<DataNode, List<DataSetRow>> dataNodeListMap = getDataSetRowMap();
- List<Callable<Void>> fillDataTasks = new LinkedList<>();
- for (Entry<DataNode, List<DataSetRow>> entry :
dataNodeListMap.entrySet()) {
- DataNode dataNode = entry.getKey();
- List<DataSetRow> dataSetRows = entry.getValue();
- DataSetMetaData dataSetMetaData = dataSet.findMetaData(dataNode);
- List<SQLValueGroup> sqlValueGroups = new LinkedList<>();
- for (DataSetRow row : dataSetRows) {
- sqlValueGroups.add(new SQLValueGroup(dataSetMetaData,
row.splitValues(DATA_COLUMN_DELIMITER)));
- }
- String insertSQL;
- DatabaseType databaseType;
- try (Connection connection =
dataSourceMap.get(dataNode.getDataSourceName()).getConnection()) {
- databaseType =
DatabaseTypeFactory.get(connection.getMetaData());
- insertSQL = generateInsertSQL(dataNode.getTableName(),
dataSetMetaData.getColumns(), databaseType);
- }
- fillDataTasks.add(new
InsertTask(dataSourceMap.get(dataNode.getDataSourceName()), insertSQL,
sqlValueGroups, databaseType));
- }
+ public void fillData(final Collection<String> tableNames) {
+ List<Callable<Void>> fillDataTasks =
createFillDataTasks(getResetPlan(tableNames));
List<Future<Void>> futures =
EXECUTOR_SERVICE_MANAGER.getExecutorService().invokeAll(fillDataTasks);
for (Future<Void> future : futures) {
future.get();
}
}
- private Map<DataNode, List<DataSetRow>> getDataSetRowMap() {
- Map<DataNode, List<DataSetRow>> result = new
LinkedHashMap<>(dataSet.getRows().size(), 1F);
+ private List<Callable<Void>> createFillDataTasks(final ResetPlan
resetPlan) throws SQLException {
+ List<Callable<Void>> result = new LinkedList<>();
+ for (Entry<DataNode, InsertDataNodePlan> entry :
resetPlan.getInsertDataNodePlans().entrySet()) {
+ DataNode dataNode = entry.getKey();
+ DataSource dataSource =
dataSourceMap.get(dataNode.getDataSourceName());
+ DatabaseType databaseType =
getDatabaseType(dataNode.getDataSourceName(), dataSource);
+ InsertDataNodePlan insertDataNodePlan = entry.getValue();
+ result.add(new InsertTask(dataSource,
getInsertSQL(dataNode.getTableName(), insertDataNodePlan.getColumnMetaData(),
databaseType),
+ insertDataNodePlan.getSqlValueGroups(), databaseType));
+ }
+ return result;
+ }
+
+ private String getInsertSQL(final String tableName, final
Collection<DataSetColumn> columnMetaData, final DatabaseType databaseType) {
+ return
INSERT_SQL_CACHE.computeIfAbsent(getInsertSQLCacheKey(tableName,
columnMetaData, databaseType), unused -> generateInsertSQL(tableName,
columnMetaData, databaseType));
+ }
+
+ private String getInsertSQLCacheKey(final String tableName, final
Collection<DataSetColumn> columnMetaData, final DatabaseType databaseType) {
+ StringBuilder result = new
StringBuilder(databaseType.getType()).append(':').append(tableName);
+ for (DataSetColumn each : columnMetaData) {
+
result.append(':').append(each.getName()).append('#').append(each.getType());
+ }
+ return result.toString();
+ }
+
+ private Map<DataNode, InsertDataNodePlan> createInsertDataNodePlans(final
Collection<String> tableNames, final Map<DataNode, DataSetMetaData>
metaDataMap) {
+ Map<DataNode, InsertDataNodePlan> result = new
LinkedHashMap<>(dataSet.getRows().size(), 1F);
for (DataSetRow each : dataSet.getRows()) {
- // The data type of the current table is currently only used by
mysql.
if (each.getDataNode().contains("t_product_extend") &&
!"MySQL".equals(databaseType.getType())) {
continue;
}
DataNode dataNode = new DataNode(each.getDataNode());
+ if (!isMatchedDataNode(dataNode, tableNames)) {
+ continue;
+ }
+ DataSetMetaData dataSetMetaData = metaDataMap.get(dataNode);
+ if (null == dataSetMetaData) {
+ throw new IllegalArgumentException(String.format("Cannot find
data node: %s", dataNode));
+ }
if (!result.containsKey(dataNode)) {
- result.put(dataNode, new LinkedList<>());
+ result.put(dataNode, new
InsertDataNodePlan(dataSetMetaData.getColumns()));
}
- result.get(dataNode).add(each);
+ result.get(dataNode).addSQLValueGroup(new
SQLValueGroup(dataSetMetaData, each.splitValues(DATA_COLUMN_DELIMITER)));
}
return result;
}
@@ -147,45 +177,174 @@ public final class DataSetEnvironmentManager {
}
/**
- * Clean data.
+ * Clean data for specified tables, or all tables when table names are
empty.
+ *
+ * @param tableNames table names
*/
- @SneakyThrows({InterruptedException.class, ExecutionException.class})
- public void cleanData() {
- List<Callable<Void>> deleteTasks = new LinkedList<>();
- for (Entry<String, Collection<String>> entry :
getDataNodeMap().entrySet()) {
- deleteTasks.add(new DeleteTask(dataSourceMap.get(entry.getKey()),
entry.getValue()));
- }
+ @SneakyThrows({SQLException.class, InterruptedException.class,
ExecutionException.class})
+ public void cleanData(final Collection<String> tableNames) {
+ List<Callable<Void>> deleteTasks =
createDeleteTasks(getResetPlan(tableNames));
List<Future<Void>> futures =
EXECUTOR_SERVICE_MANAGER.getExecutorService().invokeAll(deleteTasks);
for (Future<Void> future : futures) {
future.get();
}
}
- private Map<String, Collection<String>> getDataNodeMap() {
- Map<String, Collection<String>> result = new LinkedHashMap<>();
+ private List<Callable<Void>> createDeleteTasks(final ResetPlan resetPlan)
throws SQLException {
+ List<Callable<Void>> result = new LinkedList<>();
+ for (Entry<String, Collection<String>> entry :
resetPlan.getTableNamesByDataSourceName().entrySet()) {
+ DatabaseType databaseType = getDatabaseType(entry.getKey(),
dataSourceMap.get(entry.getKey()));
+ Collection<String> truncateSQLs = new LinkedList<>();
+ for (String each : entry.getValue()) {
+ truncateSQLs.add(getTruncateSQL(each, databaseType));
+ }
+ result.add(new DeleteTask(dataSourceMap.get(entry.getKey()),
truncateSQLs));
+ }
+ return result;
+ }
+
+ private String getTruncateSQL(final String tableName, final DatabaseType
databaseType) {
+ return TRUNCATE_SQL_CACHE.computeIfAbsent(databaseType.getType() + ':'
+ tableName, unused -> String.format("TRUNCATE TABLE %s",
getQuotedTableName(tableName, databaseType)));
+ }
+
+ private static String getQuotedTableName(final String tableName, final
DatabaseType databaseType) {
+ DatabaseTypeRegistry databaseTypeRegistry = new
DatabaseTypeRegistry(databaseType);
+ return
databaseTypeRegistry.getDialectDatabaseMetaData().getQuoteCharacter().wrap(databaseTypeRegistry.formatIdentifierPattern(tableName));
+ }
+
+ private ResetPlan getResetPlan(final Collection<String> tableNames) {
+ return
RESET_PLAN_CACHE.computeIfAbsent(getResetPlanCacheKey(tableNames), unused ->
createResetPlan(tableNames));
+ }
+
+ private String getResetPlanCacheKey(final Collection<String> tableNames) {
+ return dataSetFile + ':' + databaseType.getType() + ':' +
String.join(",", new TreeSet<>(getEffectiveTableNames(tableNames)));
+ }
+
+ private ResetPlan createResetPlan(final Collection<String> tableNames) {
+ Collection<String> effectiveTableNames =
getEffectiveTableNames(tableNames);
+ if (!effectiveTableNames.isEmpty() &&
!containsMatchedMetaData(effectiveTableNames)) {
+ effectiveTableNames = Collections.emptySet();
+ }
+ Map<String, Collection<String>> tableNamesByDataSourceName = new
LinkedHashMap<>();
+ Map<DataNode, DataSetMetaData> metaDataMap =
createMetaDataMap(effectiveTableNames, tableNamesByDataSourceName);
+ return new ResetPlan(createInsertDataNodePlans(effectiveTableNames,
metaDataMap), tableNamesByDataSourceName);
+ }
+
+ private Map<DataNode, DataSetMetaData> createMetaDataMap(final
Collection<String> tableNames, final Map<String, Collection<String>>
tableNamesByDataSourceName) {
+ Map<DataNode, DataSetMetaData> result = new LinkedHashMap<>();
for (DataSetMetaData each : dataSet.getMetaDataList()) {
- for (Entry<String, Collection<String>> entry :
getDataNodeMap(each).entrySet()) {
- if (!result.containsKey(entry.getKey())) {
- result.put(entry.getKey(), new LinkedList<>());
- }
- result.get(entry.getKey()).addAll(entry.getValue());
+ if (!isMatchedMetaData(each, tableNames)) {
+ continue;
+ }
+ for (String dataNodeText :
InlineExpressionParserFactory.newInstance(each.getDataNodes()).splitAndEvaluate())
{
+ DataNode dataNode = new DataNode(dataNodeText);
+ result.put(dataNode, each);
+ addTableName(tableNamesByDataSourceName, dataNode);
}
}
return result;
}
- private Map<String, Collection<String>> getDataNodeMap(final
DataSetMetaData dataSetMetaData) {
- Map<String, Collection<String>> result = new LinkedHashMap<>();
+ private void addTableName(final Map<String, Collection<String>>
tableNamesByDataSourceName, final DataNode dataNode) {
+ if
(!tableNamesByDataSourceName.containsKey(dataNode.getDataSourceName())) {
+ tableNamesByDataSourceName.put(dataNode.getDataSourceName(), new
LinkedList<>());
+ }
+
tableNamesByDataSourceName.get(dataNode.getDataSourceName()).add(dataNode.getTableName());
+ }
+
+ private Collection<String> getEffectiveTableNames(final Collection<String>
tableNames) {
+ return tableNames.stream().filter(each -> null != each &&
!each.isEmpty()).map(each ->
each.toLowerCase(Locale.ENGLISH)).collect(Collectors.toSet());
+ }
+
+ private boolean isMatchedMetaData(final DataSetMetaData dataSetMetaData,
final Collection<String> tableNames) {
+ if (tableNames.isEmpty()) {
+ return true;
+ }
+ if (null != dataSetMetaData.getTableName() &&
tableNames.contains(dataSetMetaData.getTableName().toLowerCase(Locale.ENGLISH)))
{
+ return true;
+ }
for (String each :
InlineExpressionParserFactory.newInstance(dataSetMetaData.getDataNodes()).splitAndEvaluate())
{
- DataNode dataNode = new DataNode(each);
- if (!result.containsKey(dataNode.getDataSourceName())) {
- result.put(dataNode.getDataSourceName(), new LinkedList<>());
+ if (isMatchedDataNode(new DataNode(each), tableNames)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean containsMatchedMetaData(final Collection<String>
tableNames) {
+ for (DataSetMetaData each : dataSet.getMetaDataList()) {
+ if (isMatchedMetaData(each, tableNames)) {
+ return true;
}
-
result.get(dataNode.getDataSourceName()).add(dataNode.getTableName());
}
+ return false;
+ }
+
+ private boolean isMatchedDataNode(final DataNode dataNode, final
Collection<String> tableNames) {
+ return tableNames.isEmpty() ||
getPossibleTableNames(dataNode.getTableName()).stream().anyMatch(tableNames::contains);
+ }
+
+ private Collection<String> getPossibleTableNames(final String tableName) {
+ Collection<String> result = new HashSet<>();
+ result.add(tableName.toLowerCase(Locale.ENGLISH));
+ result.add(tableName.replaceFirst("_[0-9]+$",
"").toLowerCase(Locale.ENGLISH));
+ result.add(tableName.replaceFirst("[0-9]+$",
"").toLowerCase(Locale.ENGLISH));
return result;
}
+ private DatabaseType getDatabaseType(final String dataSourceName, final
DataSource dataSource) throws SQLException {
+ String cacheKey = getDataSourceDatabaseTypeCacheKey(dataSourceName,
dataSource);
+ DatabaseType result = DATA_SOURCE_DATABASE_TYPE_CACHE.get(cacheKey);
+ if (null != result) {
+ return result;
+ }
+ try (Connection connection = dataSource.getConnection()) {
+ result = DatabaseTypeFactory.get(connection.getMetaData());
+ DATA_SOURCE_DATABASE_TYPE_CACHE.put(cacheKey, result);
+ return result;
+ }
+ }
+
+ private String getDataSourceDatabaseTypeCacheKey(final String
dataSourceName, final DataSource dataSource) {
+ return databaseType.getType() + ':' + dataSourceName + ':' +
System.identityHashCode(dataSource);
+ }
+
+ @RequiredArgsConstructor
+ private static final class ResetPlan {
+
+ private final Map<DataNode, InsertDataNodePlan> insertDataNodePlans;
+
+ private final Map<String, Collection<String>>
tableNamesByDataSourceName;
+
+ private Map<DataNode, InsertDataNodePlan> getInsertDataNodePlans() {
+ return insertDataNodePlans;
+ }
+
+ private Map<String, Collection<String>>
getTableNamesByDataSourceName() {
+ return tableNamesByDataSourceName;
+ }
+ }
+
+ @RequiredArgsConstructor
+ private static final class InsertDataNodePlan {
+
+ private final Collection<DataSetColumn> columnMetaData;
+
+ private final Collection<SQLValueGroup> sqlValueGroups = new
LinkedList<>();
+
+ private Collection<DataSetColumn> getColumnMetaData() {
+ return columnMetaData;
+ }
+
+ private Collection<SQLValueGroup> getSqlValueGroups() {
+ return sqlValueGroups;
+ }
+
+ private void addSQLValueGroup(final SQLValueGroup sqlValueGroup) {
+ sqlValueGroups.add(sqlValueGroup);
+ }
+ }
+
@RequiredArgsConstructor
private static final class InsertTask implements Callable<Void> {
@@ -243,25 +402,18 @@ public final class DataSetEnvironmentManager {
private final DataSource dataSource;
- private final Collection<String> tableNames;
+ private final Collection<String> truncateSQLs;
@Override
public Void call() throws SQLException {
try (Connection connection = dataSource.getConnection()) {
- DatabaseType databaseType =
DatabaseTypeFactory.get(connection.getMetaData());
- for (String each : tableNames) {
- String quotedTableName = getQuotedTableName(each,
databaseType);
- try (PreparedStatement preparedStatement =
connection.prepareStatement(String.format("TRUNCATE TABLE %s",
quotedTableName))) {
+ for (String each : truncateSQLs) {
+ try (PreparedStatement preparedStatement =
connection.prepareStatement(each)) {
preparedStatement.execute();
}
}
}
return null;
}
-
- private String getQuotedTableName(final String tableName, final
DatabaseType databaseType) {
- DatabaseTypeRegistry databaseTypeRegistry = new
DatabaseTypeRegistry(databaseType);
- return
databaseTypeRegistry.getDialectDatabaseMetaData().getQuoteCharacter().wrap(databaseTypeRegistry.formatIdentifierPattern(tableName));
- }
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetResetScopeCalculator.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetResetScopeCalculator.java
new file mode 100644
index 00000000000..158ef96a961
--- /dev/null
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/DataSetResetScopeCalculator.java
@@ -0,0 +1,479 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.test.e2e.sql.env;
+
+import org.antlr.v4.runtime.misc.ParseCancellationException;
+import
org.apache.shardingsphere.database.exception.core.exception.syntax.sql.DialectSQLParsingException;
+import org.apache.shardingsphere.database.connector.core.type.DatabaseType;
+import org.apache.shardingsphere.infra.datanode.DataNode;
+import
org.apache.shardingsphere.infra.expr.entry.InlineExpressionParserFactory;
+import org.apache.shardingsphere.infra.parser.SQLParserEngine;
+import org.apache.shardingsphere.infra.parser.ShardingSphereSQLParserEngine;
+import org.apache.shardingsphere.sql.parser.engine.api.CacheOption;
+import
org.apache.shardingsphere.sql.parser.engine.exception.SQLParsingException;
+import
org.apache.shardingsphere.sql.parser.statement.core.extractor.TableExtractor;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.assignment.ColumnAssignmentSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.assignment.InsertValuesSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.assignment.SetAssignmentSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.column.OnDuplicateKeyColumnsSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.BetweenExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.BinaryOperationExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.CaseWhenExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.CollateExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ExistsSubqueryExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ExpressionSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.FunctionSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.InExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.KeyValueSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ListExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.NotExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.QuantifySubqueryExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.RowExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.TypeCastExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.UnaryOperationExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.ValuesExpression;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.complex.CommonTableExpressionSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.subquery.SubqueryExpressionSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.expr.subquery.SubquerySegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.table.MultiTableConditionalIntoSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.dml.table.MultiTableInsertIntoSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.WithSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.DeleteMultiTableSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.JoinTableSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SimpleTableSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.SubqueryTableSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.segment.generic.table.TableSegment;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.SQLStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.DeleteStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.InsertStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.MergeStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.SelectStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.statement.type.dml.UpdateStatement;
+import
org.apache.shardingsphere.sql.parser.statement.core.util.MultiSQLSplitter;
+import
org.apache.shardingsphere.test.e2e.sql.cases.casse.assertion.SQLE2ETestCaseAssertion;
+import
org.apache.shardingsphere.test.e2e.sql.cases.casse.assertion.SQLE2ETestCaseAssertionSQL;
+import org.apache.shardingsphere.test.e2e.sql.cases.dataset.DataSetLoader;
+import
org.apache.shardingsphere.test.e2e.sql.cases.dataset.metadata.DataSetMetaData;
+import
org.apache.shardingsphere.test.e2e.sql.framework.param.model.AssertionTestParameter;
+import
org.apache.shardingsphere.test.e2e.sql.framework.param.model.CaseTestParameter;
+import
org.apache.shardingsphere.test.e2e.sql.framework.param.model.E2ETestParameter;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Calculator for data set reset table scope.
+ */
+public final class DataSetResetScopeCalculator {
+
+ private static final CacheOption CACHE_OPTION = new CacheOption(128,
1024L);
+
+ private static final Map<String, SQLParserEngine> SQL_PARSER_ENGINE_MAP =
new ConcurrentHashMap<>();
+
+ private static final Map<String, Collection<String>>
RESET_TABLE_NAMES_CACHE = new ConcurrentHashMap<>();
+
+ /**
+ * Get reset table names.
+ *
+ * @param testParam test parameter
+ * @return reset table names, or empty collection for full reset fallback
+ */
+ public Collection<String> getResetTableNames(final E2ETestParameter
testParam) {
+ return
RESET_TABLE_NAMES_CACHE.computeIfAbsent(getResetTableNamesCacheKey(testParam),
unused -> calculateResetTableNames(testParam));
+ }
+
+ private Collection<String> calculateResetTableNames(final E2ETestParameter
testParam) {
+ Collection<String> result = new LinkedHashSet<>();
+ if (!addParsedTableNames(result,
testParam.getTestCaseContext().getTestCase().getSql(),
testParam.getDatabaseType())) {
+ return Collections.emptyList();
+ }
+ for (SQLE2ETestCaseAssertion each : getAssertions(testParam)) {
+ if (!addAssertionScope(result, testParam, each)) {
+ return Collections.emptyList();
+ }
+ }
+ return result;
+ }
+
+ private String getResetTableNamesCacheKey(final E2ETestParameter
testParam) {
+ StringBuilder result = new StringBuilder();
+ appendCacheKeyValue(result, testParam.getKey());
+ appendCacheKeyValue(result, testParam.getMode().name());
+ appendCacheKeyValue(result,
testParam.getTestCaseContext().getParentPath());
+ appendCacheKeyValue(result,
testParam.getTestCaseContext().getTestCase().getSql());
+ for (SQLE2ETestCaseAssertion each : getAssertions(testParam)) {
+ appendAssertionCacheKey(result, each);
+ }
+ return result.toString();
+ }
+
+ private void appendAssertionCacheKey(final StringBuilder cacheKey, final
SQLE2ETestCaseAssertion assertion) {
+ if (null == assertion) {
+ appendCacheKeyValue(cacheKey, "");
+ return;
+ }
+ appendCacheKeyValue(cacheKey, assertion.getExpectedDataFile());
+ appendCacheKeyValue(cacheKey,
assertion.getExpectedGeneratedKeyDataFile());
+ appendSQLCacheKey(cacheKey, assertion.getInitialSQL());
+ appendSQLCacheKey(cacheKey, assertion.getAssertionSQL());
+ }
+
+ private void appendSQLCacheKey(final StringBuilder cacheKey, final
SQLE2ETestCaseAssertionSQL sql) {
+ appendCacheKeyValue(cacheKey, null == sql ? "" : sql.getSql());
+ }
+
+ private void appendCacheKeyValue(final StringBuilder cacheKey, final
String value) {
+ String actualValue = null == value ? "" : value;
+ cacheKey.append(actualValue.length()).append(':').append(actualValue);
+ }
+
+ private boolean addAssertionScope(final Collection<String> tableNames,
final E2ETestParameter testParam, final SQLE2ETestCaseAssertion assertion) {
+ if (null == assertion) {
+ return true;
+ }
+ if (!addSQLTableNames(tableNames, assertion.getInitialSQL(),
testParam.getDatabaseType()) || !addSQLTableNames(tableNames,
assertion.getAssertionSQL(), testParam.getDatabaseType())) {
+ return false;
+ }
+ return addDataSetTableNames(tableNames, testParam,
assertion.getExpectedDataFile());
+ }
+
+ private Collection<SQLE2ETestCaseAssertion> getAssertions(final
E2ETestParameter testParam) {
+ if (testParam instanceof AssertionTestParameter) {
+ return Collections.singleton(((AssertionTestParameter)
testParam).getAssertion());
+ }
+ if (testParam instanceof CaseTestParameter) {
+ return
testParam.getTestCaseContext().getTestCase().getAssertions();
+ }
+ return Collections.emptyList();
+ }
+
+ private boolean addSQLTableNames(final Collection<String> tableNames,
final SQLE2ETestCaseAssertionSQL sql, final DatabaseType databaseType) {
+ return null == sql || null == sql.getSql() ||
addParsedTableNames(tableNames, sql.getSql(), databaseType);
+ }
+
+ private boolean addParsedTableNames(final Collection<String> tableNames,
final String sql, final DatabaseType databaseType) {
+ if (null == sql || sql.isEmpty()) {
+ return true;
+ }
+ for (String each : MultiSQLSplitter.split(sql)) {
+ if (!addParsedSingleSQLTableNames(tableNames, each, databaseType))
{
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean addParsedSingleSQLTableNames(final Collection<String>
tableNames, final String sql, final DatabaseType databaseType) {
+ try {
+ SQLStatement sqlStatement =
getSQLParserEngine(databaseType).parse(sql, true);
+ if (isUnextractedWithSQL(sql, sqlStatement)) {
+ return false;
+ }
+ Collection<String> virtualTableNames = new LinkedHashSet<>();
+ Collection<String> parsedTableNames = new LinkedHashSet<>();
+ TableExtractor tableExtractor = new TableExtractor();
+ tableExtractor.extractTablesFromSQLStatement(sqlStatement);
+ addExtractedRewriteTableNames(parsedTableNames, tableExtractor);
+ tableExtractor.getTableContext().forEach(each ->
addVirtualTableNames(virtualTableNames, each));
+ addSupplementalTableNames(parsedTableNames, virtualTableNames,
sqlStatement);
+ removeVirtualTableNames(parsedTableNames, virtualTableNames);
+ tableNames.addAll(parsedTableNames);
+ return true;
+ } catch (final SQLParsingException | ParseCancellationException |
DialectSQLParsingException ignored) {
+ return isTransactionControlSQL(sql);
+ }
+ }
+
+ private void addSupplementalTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final SQLStatement
sqlStatement) {
+ if (sqlStatement instanceof InsertStatement) {
+ addInsertStatementTableNames(tableNames, virtualTableNames,
(InsertStatement) sqlStatement);
+ } else if (sqlStatement instanceof UpdateStatement) {
+ addUpdateStatementTableNames(tableNames, virtualTableNames,
(UpdateStatement) sqlStatement);
+ } else if (sqlStatement instanceof DeleteStatement) {
+ addDeleteStatementTableNames(tableNames, virtualTableNames,
(DeleteStatement) sqlStatement);
+ } else if (sqlStatement instanceof MergeStatement) {
+ addMergeStatementTableNames(tableNames, virtualTableNames,
(MergeStatement) sqlStatement);
+ } else if (sqlStatement instanceof SelectStatement) {
+ addSelectStatementTableNames(tableNames, virtualTableNames,
(SelectStatement) sqlStatement);
+ }
+ }
+
+ private void addSelectStatementTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final SelectStatement
selectStatement) {
+ selectStatement.getWith().ifPresent(optional ->
addWithTableNames(tableNames, virtualTableNames, optional));
+ }
+
+ private void addInsertStatementTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final InsertStatement
insertStatement) {
+ insertStatement.getTable().ifPresent(optional ->
addTableSegmentTableNames(tableNames, virtualTableNames, optional));
+ insertStatement.getInsertSelect().ifPresent(optional ->
addSubqueryTableNames(tableNames, virtualTableNames, optional));
+ insertStatement.getWith().ifPresent(optional ->
addWithTableNames(tableNames, virtualTableNames, optional));
+ insertStatement.getMultiTableInsertInto().ifPresent(optional ->
addMultiTableInsertIntoTableNames(tableNames, virtualTableNames, optional));
+ insertStatement.getMultiTableConditionalInto().ifPresent(optional ->
addMultiTableConditionalIntoTableNames(tableNames, virtualTableNames,
optional));
+ insertStatement.getValues().forEach(each ->
addInsertValuesTableNames(tableNames, virtualTableNames, each));
+ insertStatement.getSetAssignment().ifPresent(optional ->
addSetAssignmentTableNames(tableNames, virtualTableNames, optional));
+ insertStatement.getOnDuplicateKeyColumns().ifPresent(optional ->
addOnDuplicateKeyColumnsTableNames(tableNames, virtualTableNames, optional));
+ }
+
+ private void addUpdateStatementTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final UpdateStatement
updateStatement) {
+ addTableSegmentTableNames(tableNames, virtualTableNames,
updateStatement.getTable());
+ updateStatement.getFrom().ifPresent(optional ->
addTableSegmentTableNames(tableNames, virtualTableNames, optional));
+ updateStatement.getWith().ifPresent(optional ->
addWithTableNames(tableNames, virtualTableNames, optional));
+ updateStatement.getAssignment().ifPresent(optional ->
addSetAssignmentTableNames(tableNames, virtualTableNames, optional));
+ }
+
+ private void addDeleteStatementTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final DeleteStatement
deleteStatement) {
+ addTableSegmentTableNames(tableNames, virtualTableNames,
deleteStatement.getTable());
+ deleteStatement.getWith().ifPresent(optional ->
addWithTableNames(tableNames, virtualTableNames, optional));
+ }
+
+ private void addMergeStatementTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final MergeStatement
mergeStatement) {
+ addTableSegmentTableNames(tableNames, virtualTableNames,
mergeStatement.getTarget());
+ addTableSegmentTableNames(tableNames, virtualTableNames,
mergeStatement.getSource());
+ mergeStatement.getUpdate().ifPresent(optional ->
addUpdateStatementTableNames(tableNames, virtualTableNames, optional));
+ mergeStatement.getInsert().ifPresent(optional ->
addInsertStatementTableNames(tableNames, virtualTableNames, optional));
+ mergeStatement.getWith().ifPresent(optional ->
addWithTableNames(tableNames, virtualTableNames, optional));
+ }
+
+ private void addMultiTableInsertIntoTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final
MultiTableInsertIntoSegment insertIntoSegment) {
+ insertIntoSegment.getInsertStatements().forEach(each ->
addInsertStatementTableNames(tableNames, virtualTableNames, each));
+ }
+
+ private void addMultiTableConditionalIntoTableNames(final
Collection<String> tableNames, final Collection<String> virtualTableNames,
+ final
MultiTableConditionalIntoSegment conditionalIntoSegment) {
+ conditionalIntoSegment.getWhenThenSegments()
+ .forEach(each ->
each.getThenSegment().getInsertStatements().forEach(optional ->
addInsertStatementTableNames(tableNames, virtualTableNames, optional)));
+ conditionalIntoSegment.getElseSegment().ifPresent(optional ->
optional.getInsertStatements().forEach(each ->
addInsertStatementTableNames(tableNames, virtualTableNames, each)));
+ }
+
+ private void addInsertValuesTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final
InsertValuesSegment valuesSegment) {
+ addExpressionTableNames(tableNames, virtualTableNames,
valuesSegment.getValues());
+ }
+
+ private void addSetAssignmentTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final
SetAssignmentSegment setAssignmentSegment) {
+ addColumnAssignmentTableNames(tableNames, virtualTableNames,
setAssignmentSegment.getAssignments());
+ }
+
+ private void addOnDuplicateKeyColumnsTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final
OnDuplicateKeyColumnsSegment onDuplicateKeyColumnsSegment) {
+ addColumnAssignmentTableNames(tableNames, virtualTableNames,
onDuplicateKeyColumnsSegment.getColumns());
+ }
+
+ private void addColumnAssignmentTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final
Collection<ColumnAssignmentSegment> assignments) {
+ assignments.forEach(each -> addExpressionTableNames(tableNames,
virtualTableNames, each.getValue()));
+ }
+
+ private void addExpressionTableNames(final Collection<String> tableNames,
final Collection<String> virtualTableNames, final Collection<ExpressionSegment>
expressionSegments) {
+ expressionSegments.forEach(each -> addExpressionTableNames(tableNames,
virtualTableNames, each));
+ }
+
+ private void addExpressionTableNames(final Collection<String> tableNames,
final Collection<String> virtualTableNames, final ExpressionSegment
expressionSegment) {
+ if (null == expressionSegment) {
+ return;
+ }
+ if (expressionSegment instanceof SubquerySegment) {
+ addSubqueryTableNames(tableNames, virtualTableNames,
(SubquerySegment) expressionSegment);
+ } else if (expressionSegment instanceof SubqueryExpressionSegment) {
+ addSubqueryTableNames(tableNames, virtualTableNames,
((SubqueryExpressionSegment) expressionSegment).getSubquery());
+ } else if (expressionSegment instanceof ExistsSubqueryExpression) {
+ addSubqueryTableNames(tableNames, virtualTableNames,
((ExistsSubqueryExpression) expressionSegment).getSubquery());
+ } else if (expressionSegment instanceof QuantifySubqueryExpression) {
+ addSubqueryTableNames(tableNames, virtualTableNames,
((QuantifySubqueryExpression) expressionSegment).getSubquery());
+ } else {
+ addNestedExpressionTableNames(tableNames, virtualTableNames,
expressionSegment);
+ }
+ }
+
+ private void addNestedExpressionTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final ExpressionSegment
expressionSegment) {
+ if (expressionSegment instanceof BinaryOperationExpression) {
+ BinaryOperationExpression binaryOperationExpression =
(BinaryOperationExpression) expressionSegment;
+ addExpressionTableNames(tableNames, virtualTableNames,
binaryOperationExpression.getLeft());
+ addExpressionTableNames(tableNames, virtualTableNames,
binaryOperationExpression.getRight());
+ } else if (expressionSegment instanceof FunctionSegment) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((FunctionSegment) expressionSegment).getParameters());
+ } else if (expressionSegment instanceof ListExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((ListExpression) expressionSegment).getItems());
+ } else if (expressionSegment instanceof RowExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((RowExpression) expressionSegment).getItems());
+ } else if (expressionSegment instanceof InExpression) {
+ InExpression inExpression = (InExpression) expressionSegment;
+ addExpressionTableNames(tableNames, virtualTableNames,
inExpression.getLeft());
+ addExpressionTableNames(tableNames, virtualTableNames,
inExpression.getRight());
+ } else if (expressionSegment instanceof BetweenExpression) {
+ BetweenExpression betweenExpression = (BetweenExpression)
expressionSegment;
+ addExpressionTableNames(tableNames, virtualTableNames,
betweenExpression.getLeft());
+ addExpressionTableNames(tableNames, virtualTableNames,
betweenExpression.getBetweenExpr());
+ addExpressionTableNames(tableNames, virtualTableNames,
betweenExpression.getAndExpr());
+ } else if (expressionSegment instanceof CaseWhenExpression) {
+ addCaseWhenTableNames(tableNames, virtualTableNames,
(CaseWhenExpression) expressionSegment);
+ } else {
+ addOtherExpressionTableNames(tableNames, virtualTableNames,
expressionSegment);
+ }
+ }
+
+ private void addCaseWhenTableNames(final Collection<String> tableNames,
final Collection<String> virtualTableNames, final CaseWhenExpression
caseWhenExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
caseWhenExpression.getCaseExpr());
+ addExpressionTableNames(tableNames, virtualTableNames,
caseWhenExpression.getWhenExprs());
+ addExpressionTableNames(tableNames, virtualTableNames,
caseWhenExpression.getThenExprs());
+ addExpressionTableNames(tableNames, virtualTableNames,
caseWhenExpression.getElseExpr());
+ }
+
+ private void addOtherExpressionTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final ExpressionSegment
expressionSegment) {
+ if (expressionSegment instanceof TypeCastExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((TypeCastExpression) expressionSegment).getExpression());
+ } else if (expressionSegment instanceof UnaryOperationExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((UnaryOperationExpression) expressionSegment).getExpression());
+ } else if (expressionSegment instanceof NotExpression) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((NotExpression) expressionSegment).getExpression());
+ } else if (expressionSegment instanceof CollateExpression) {
+ ((CollateExpression)
expressionSegment).getExpr().ifPresent(optional ->
addExpressionTableNames(tableNames, virtualTableNames, optional));
+ } else if (expressionSegment instanceof ValuesExpression) {
+ ((ValuesExpression)
expressionSegment).getRowConstructorList().forEach(each ->
addInsertValuesTableNames(tableNames, virtualTableNames, each));
+ } else if (expressionSegment instanceof KeyValueSegment) {
+ KeyValueSegment keyValueSegment = (KeyValueSegment)
expressionSegment;
+ addExpressionTableNames(tableNames, virtualTableNames,
keyValueSegment.getKey());
+ addExpressionTableNames(tableNames, virtualTableNames,
keyValueSegment.getValue());
+ } else if (expressionSegment instanceof ColumnAssignmentSegment) {
+ addExpressionTableNames(tableNames, virtualTableNames,
((ColumnAssignmentSegment) expressionSegment).getValue());
+ }
+ }
+
+ private void addWithTableNames(final Collection<String> tableNames, final
Collection<String> virtualTableNames, final WithSegment withSegment) {
+ for (CommonTableExpressionSegment each :
withSegment.getCommonTableExpressions()) {
+ each.getAliasName().ifPresent(optional ->
virtualTableNames.add(optional.toLowerCase(Locale.ENGLISH)));
+ addSubqueryTableNames(tableNames, virtualTableNames,
each.getSubquery());
+ }
+ }
+
+ private void addTableSegmentTableNames(final Collection<String>
tableNames, final Collection<String> virtualTableNames, final TableSegment
tableSegment) {
+ addVirtualTableNames(virtualTableNames, tableSegment);
+ if (tableSegment instanceof SimpleTableSegment) {
+ addTableName(tableNames, (SimpleTableSegment) tableSegment);
+ } else if (tableSegment instanceof JoinTableSegment) {
+ JoinTableSegment joinTableSegment = (JoinTableSegment)
tableSegment;
+ addTableSegmentTableNames(tableNames, virtualTableNames,
joinTableSegment.getLeft());
+ addTableSegmentTableNames(tableNames, virtualTableNames,
joinTableSegment.getRight());
+ } else if (tableSegment instanceof SubqueryTableSegment) {
+ addSubqueryTableNames(tableNames, virtualTableNames,
((SubqueryTableSegment) tableSegment).getSubquery());
+ } else if (tableSegment instanceof DeleteMultiTableSegment) {
+ DeleteMultiTableSegment deleteMultiTableSegment =
(DeleteMultiTableSegment) tableSegment;
+ deleteMultiTableSegment.getActualDeleteTables().forEach(each ->
addTableName(tableNames, each));
+ addTableSegmentTableNames(tableNames, virtualTableNames,
deleteMultiTableSegment.getRelationTable());
+ }
+ }
+
+ private void addSubqueryTableNames(final Collection<String> tableNames,
final Collection<String> virtualTableNames, final SubquerySegment
subquerySegment) {
+ if (null != subquerySegment.getSelect()) {
+ TableExtractor tableExtractor = new TableExtractor();
+
tableExtractor.extractTablesFromSelect(subquerySegment.getSelect());
+ tableExtractor.getTableContext().forEach(each ->
addVirtualTableNames(virtualTableNames, each));
+ addSelectStatementTableNames(tableNames, virtualTableNames,
subquerySegment.getSelect());
+ addExtractedRewriteTableNames(tableNames, tableExtractor);
+ }
+ if (null != subquerySegment.getMerge()) {
+ addMergeStatementTableNames(tableNames, virtualTableNames,
subquerySegment.getMerge());
+ }
+ }
+
+ private void addExtractedRewriteTableNames(final Collection<String>
tableNames, final TableExtractor tableExtractor) {
+ for (SimpleTableSegment each : tableExtractor.getRewriteTables()) {
+ addTableName(tableNames, each);
+ }
+ }
+
+ private void addTableName(final Collection<String> tableNames, final
SimpleTableSegment tableSegment) {
+ tableNames.add(tableSegment.getTableName().getIdentifier().getValue());
+ }
+
+ private void addVirtualTableNames(final Collection<String>
virtualTableNames, final TableSegment tableSegment) {
+ if (tableSegment instanceof SimpleTableSegment) {
+ SimpleTableSegment simpleTableSegment = (SimpleTableSegment)
tableSegment;
+ simpleTableSegment.getAliasName().filter(optional ->
!optional.equalsIgnoreCase(simpleTableSegment.getTableName().getIdentifier().getValue()))
+ .ifPresent(optional ->
virtualTableNames.add(optional.toLowerCase(Locale.ENGLISH)));
+ } else if (tableSegment instanceof JoinTableSegment) {
+ JoinTableSegment joinTableSegment = (JoinTableSegment)
tableSegment;
+ joinTableSegment.getAliasName().ifPresent(optional ->
virtualTableNames.add(optional.toLowerCase(Locale.ENGLISH)));
+ addVirtualTableNames(virtualTableNames,
joinTableSegment.getLeft());
+ addVirtualTableNames(virtualTableNames,
joinTableSegment.getRight());
+ } else if (tableSegment instanceof SubqueryTableSegment) {
+ tableSegment.getAliasName().ifPresent(optional ->
virtualTableNames.add(optional.toLowerCase(Locale.ENGLISH)));
+ }
+ }
+
+ private void removeVirtualTableNames(final Collection<String> tableNames,
final Collection<String> virtualTableNames) {
+ tableNames.removeIf(each ->
virtualTableNames.contains(each.toLowerCase(Locale.ENGLISH)));
+ }
+
+ private boolean isUnextractedWithSQL(final String sql, final SQLStatement
sqlStatement) {
+ return sql.trim().toUpperCase(Locale.ENGLISH).startsWith("WITH") &&
!containsWithSegment(sqlStatement);
+ }
+
+ private boolean containsWithSegment(final SQLStatement sqlStatement) {
+ if (sqlStatement instanceof SelectStatement) {
+ return ((SelectStatement) sqlStatement).getWith().isPresent();
+ }
+ if (sqlStatement instanceof InsertStatement) {
+ return ((InsertStatement) sqlStatement).getWith().isPresent();
+ }
+ if (sqlStatement instanceof UpdateStatement) {
+ return ((UpdateStatement) sqlStatement).getWith().isPresent();
+ }
+ if (sqlStatement instanceof DeleteStatement) {
+ return ((DeleteStatement) sqlStatement).getWith().isPresent();
+ }
+ return sqlStatement instanceof MergeStatement && ((MergeStatement)
sqlStatement).getWith().isPresent();
+ }
+
+ private boolean isTransactionControlSQL(final String sql) {
+ String trimSQL = sql.trim().toUpperCase(Locale.ENGLISH);
+ return trimSQL.startsWith("BEGIN") || trimSQL.startsWith("COMMIT") ||
trimSQL.startsWith("ROLLBACK") || trimSQL.startsWith("START TRANSACTION")
+ || trimSQL.startsWith("SAVEPOINT") ||
trimSQL.startsWith("RELEASE SAVEPOINT") || trimSQL.startsWith("SET AUTOCOMMIT");
+ }
+
+ private SQLParserEngine getSQLParserEngine(final DatabaseType
databaseType) {
+ return SQL_PARSER_ENGINE_MAP.computeIfAbsent(databaseType.getType(),
unused -> new ShardingSphereSQLParserEngine(databaseType, CACHE_OPTION,
CACHE_OPTION));
+ }
+
+ private boolean addDataSetTableNames(final Collection<String> tableNames,
final E2ETestParameter testParam, final String expectedDataFile) {
+ if (null == expectedDataFile) {
+ return true;
+ }
+ try {
+ for (DataSetMetaData each :
DataSetLoader.load(testParam.getTestCaseContext().getParentPath(),
testParam.getScenario(), testParam.getDatabaseType(), testParam.getMode(),
expectedDataFile)
+ .getMetaDataList()) {
+ addDataSetMetaDataTableNames(tableNames, each);
+ }
+ return true;
+ } catch (final IllegalArgumentException ignored) {
+ return false;
+ }
+ }
+
+ private void addDataSetMetaDataTableNames(final Collection<String>
tableNames, final DataSetMetaData dataSetMetaData) {
+ if (null != dataSetMetaData.getTableName()) {
+ tableNames.add(dataSetMetaData.getTableName());
+ }
+ for (String each :
InlineExpressionParserFactory.newInstance(dataSetMetaData.getDataNodes()).splitAndEvaluate())
{
+ String tableName = new DataNode(each).getTableName();
+ tableNames.add(tableName);
+ tableNames.add(tableName.replaceFirst("_[0-9]+$", ""));
+ tableNames.add(tableName.replaceFirst("[0-9]+$", ""));
+ }
+ }
+}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/SQLE2EEnvironmentEngine.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/SQLE2EEnvironmentEngine.java
index 3262e4d2f2d..db29f45cae6 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/SQLE2EEnvironmentEngine.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/SQLE2EEnvironmentEngine.java
@@ -31,9 +31,9 @@ import
org.apache.shardingsphere.test.e2e.sql.env.container.compose.ContainerCom
import javax.sql.DataSource;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
/**
* SQL E2E environment engine.
@@ -43,7 +43,7 @@ public final class SQLE2EEnvironmentEngine {
private static final ContainerComposerRegistry CONTAINER_COMPOSER_REGISTRY
= new ContainerComposerRegistry();
- private static final Collection<String> INITIALIZED_SUITES = new
HashSet<>();
+ private static final Collection<String> INITIALIZED_SUITES =
ConcurrentHashMap.newKeySet();
@Getter(AccessLevel.NONE)
private final ContainerComposer containerComposer;
@@ -58,7 +58,7 @@ public final class SQLE2EEnvironmentEngine {
containerComposer =
CONTAINER_COMPOSER_REGISTRY.getContainerComposer(key, scenario, databaseType,
mode, adapter);
containerComposer.start();
actualDataSourceMap = containerComposer.getActualDataSourceMap();
- targetDataSource = containerComposer.getTargetDataSource();
+ targetDataSource =
CONTAINER_COMPOSER_REGISTRY.getTargetDataSource(key);
expectedDataSourceMap = containerComposer.getExpectedDataSourceMap();
executeLogicDatabaseInitSQLFileOnlyOnce(key, scenario, databaseType);
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/container/compose/ContainerComposerRegistry.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/container/compose/ContainerComposerRegistry.java
index 4dd926852ce..15c28e3f07a 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/container/compose/ContainerComposerRegistry.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/env/container/compose/ContainerComposerRegistry.java
@@ -40,6 +40,8 @@ public final class ContainerComposerRegistry implements
AutoCloseable {
private final Map<String, ContainerComposer> containerComposers = new
HashMap<>(7, 1F);
+ private final Map<String, DataSource> targetDataSources = new HashMap<>(7,
1F);
+
/**
* Get container composer.
*
@@ -51,9 +53,6 @@ public final class ContainerComposerRegistry implements
AutoCloseable {
* @return composed container
*/
public ContainerComposer getContainerComposer(final String key, final
String scenario, final DatabaseType databaseType, final Mode mode, final
Adapter adapter) {
- if (containerComposers.containsKey(key)) {
- return containerComposers.get(key);
- }
synchronized (containerComposers) {
if (!containerComposers.containsKey(key)) {
close();
@@ -63,6 +62,22 @@ public final class ContainerComposerRegistry implements
AutoCloseable {
}
}
+ /**
+ * Get target data source.
+ *
+ * @param key key
+ * @return target data source
+ */
+ public DataSource getTargetDataSource(final String key) {
+ synchronized (containerComposers) {
+ return targetDataSources.computeIfAbsent(key, unused -> {
+ ContainerComposer containerComposer =
containerComposers.get(key);
+ Preconditions.checkNotNull(containerComposer, "Container
composer `%s` does not exist.", key);
+ return containerComposer.getTargetDataSource();
+ });
+ }
+ }
+
private boolean isClusterMode(final Mode mode, final Adapter adapter) {
return Mode.CLUSTER == mode && Adapter.PROXY == adapter;
}
@@ -74,11 +89,14 @@ public final class ContainerComposerRegistry implements
AutoCloseable {
@Override
public void close() {
synchronized (containerComposers) {
+ for (DataSource each : targetDataSources.values()) {
+ closeTargetDataSource(each);
+ }
for (ContainerComposer each : containerComposers.values()) {
- closeTargetDataSource(each.getTargetDataSource());
closeActualDataSourceMap(each.getActualDataSourceMap());
closeContainer(each);
}
+ targetDataSources.clear();
containerComposers.clear();
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/ddl/DDLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/ddl/DDLE2EIT.java
index 000aa309fd0..a3cc20f9cba 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/ddl/DDLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/ddl/DDLE2EIT.java
@@ -44,23 +44,41 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.time.Duration;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
@SQLE2EITSettings(SQLCommandType.DDL)
@Setter
class DDLE2EIT implements SQLE2EIT {
+ private static final Duration META_DATA_WAIT_TIMEOUT =
Duration.ofSeconds(30L);
+
+ private static final Duration META_DATA_POLL_INTERVAL =
Duration.ofMillis(100L);
+
+ private static final Duration DDL_COMPLETION_DELAY =
Duration.ofMillis(1500L);
+
+ private static final Pattern CREATE_TABLE_OR_VIEW_PATTERN =
Pattern.compile("(?is)^\\s*CREATE\\s+(?:OR\\s+REPLACE\\s+)?(?:TABLE|VIEW)\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?([^\\s(]+).*");
+
+ private static final Pattern DROP_TABLE_OR_VIEW_PATTERN =
Pattern.compile("(?is)^\\s*DROP\\s+(?:TABLE|VIEW)\\s+(?:IF\\s+EXISTS\\s+)?([^\\s(;]+).*");
+
+ private static final Pattern CREATE_INDEX_PATTERN =
Pattern.compile("(?is)^\\s*CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+([^\\s(]+)\\s+ON\\s+([^\\s(]+).*");
+
+ private static final Pattern DROP_INDEX_WITH_TABLE_PATTERN =
Pattern.compile("(?is)^\\s*DROP\\s+INDEX\\s+([^\\s(]+)\\s+ON\\s+([^\\s(]+).*");
+
+ private static final Pattern DROP_INDEX_PATTERN =
Pattern.compile("(?is)^\\s*DROP\\s+INDEX\\s+([^\\s(]+).*");
+
private SQLE2EEnvironmentEngine environmentEngine;
@ParameterizedTest(name = "{0}")
@@ -79,7 +97,7 @@ class DDLE2EIT implements SQLE2EIT {
} else {
executeUpdateForPreparedStatement(context, connection);
}
- assertTableMetaData(testParam, context);
+ assertTableMetaDataEventually(testParam, context);
} finally {
tearDown(context);
}
@@ -89,14 +107,12 @@ class DDLE2EIT implements SQLE2EIT {
try (Statement statement = connection.createStatement()) {
assertFalse(statement.executeUpdate(context.getSQL()) > 0, "Not a
DDL statement.");
}
- waitCompleted();
}
private void executeUpdateForPreparedStatement(final SQLE2EITContext
context, final Connection connection) throws SQLException {
try (PreparedStatement preparedStatement =
connection.prepareStatement(context.getSQL())) {
assertFalse(preparedStatement.executeUpdate() > 0, "Not a DDL
statement.");
}
- waitCompleted();
}
@ParameterizedTest(name = "{0}")
@@ -115,7 +131,7 @@ class DDLE2EIT implements SQLE2EIT {
} else {
executeForPreparedStatement(context, connection);
}
- assertTableMetaData(testParam, context);
+ assertTableMetaDataEventually(testParam, context);
} finally {
tearDown(context);
}
@@ -125,14 +141,12 @@ class DDLE2EIT implements SQLE2EIT {
try (Statement statement = connection.createStatement()) {
assertFalse(statement.execute(context.getSQL()), "Not a DDL
statement.");
}
- waitCompleted();
}
private void executeForPreparedStatement(final SQLE2EITContext context,
final Connection connection) throws SQLException {
try (PreparedStatement preparedStatement =
connection.prepareStatement(context.getSQL())) {
assertFalse(preparedStatement.execute(), "Not a DDL statement.");
}
- waitCompleted();
}
private void init(final SQLE2EITContext context) throws SQLException {
@@ -151,7 +165,7 @@ class DDLE2EIT implements SQLE2EIT {
try (PreparedStatement preparedStatement =
connection.prepareStatement(each)) {
preparedStatement.executeUpdate();
}
- waitCompleted();
+ waitSQLCompleted(each, context);
}
}
@@ -171,14 +185,112 @@ class DDLE2EIT implements SQLE2EIT {
try (PreparedStatement preparedStatement =
connection.prepareStatement(each)) {
preparedStatement.executeUpdate();
}
- waitCompleted();
+ waitSQLCompleted(each, context);
+ }
+ }
+
+ private void waitSQLCompleted(final String sql, final SQLE2EITContext
context) {
+ Matcher createTableOrViewMatcher =
CREATE_TABLE_OR_VIEW_PATTERN.matcher(sql);
+ if (createTableOrViewMatcher.matches() && waitTableExists(context,
createTableOrViewMatcher.group(1), true)) {
+ return;
+ }
+ Matcher dropTableOrViewMatcher =
DROP_TABLE_OR_VIEW_PATTERN.matcher(sql);
+ if (dropTableOrViewMatcher.matches() && waitTableExists(context,
dropTableOrViewMatcher.group(1), false)) {
+ return;
+ }
+ Matcher createIndexMatcher = CREATE_INDEX_PATTERN.matcher(sql);
+ if (createIndexMatcher.matches() && waitIndexExists(context,
createIndexMatcher.group(2), createIndexMatcher.group(1), true)) {
+ return;
+ }
+ Matcher dropIndexWithTableMatcher =
DROP_INDEX_WITH_TABLE_PATTERN.matcher(sql);
+ if (dropIndexWithTableMatcher.matches() && waitIndexExists(context,
dropIndexWithTableMatcher.group(2), dropIndexWithTableMatcher.group(1), false))
{
+ return;
+ }
+ Matcher dropIndexMatcher = DROP_INDEX_PATTERN.matcher(sql);
+ if (dropIndexMatcher.matches() && waitIndexExists(context,
context.getAssertion().getInitialSQL().getAffectedTable(),
dropIndexMatcher.group(1), false)) {
+ return;
+ }
+ waitCompleted();
+ }
+
+ private boolean waitTableExists(final SQLE2EITContext context, final
String tableName, final boolean exists) {
+ Collection<DataNode> dataNodes = findDataNodes(context,
getIdentifierValue(tableName));
+ if (dataNodes.isEmpty()) {
+ return false;
+ }
+
Awaitility.await().atMost(META_DATA_WAIT_TIMEOUT).pollInterval(META_DATA_POLL_INTERVAL).untilAsserted(()
-> assertTableState(dataNodes, exists));
+ return true;
+ }
+
+ private void assertTableState(final Collection<DataNode> dataNodes, final
boolean exists) throws SQLException {
+ if (!exists) {
+ assertNotContainsTable(environmentEngine, dataNodes);
+ return;
+ }
+ boolean tableExists = false;
+ for (DataNode each : dataNodes) {
+ try (Connection connection =
environmentEngine.getActualDataSourceMap().get(each.getDataSourceName()).getConnection())
{
+ if (containsTable(connection, each.getTableName())) {
+ tableExists = true;
+ break;
+ }
+ }
}
+ assertTrue(tableExists, "Expected table does not exist");
+ }
+
+ private boolean waitIndexExists(final SQLE2EITContext context, final
String tableName, final String indexName, final boolean exists) {
+ Collection<DataNode> dataNodes = findDataNodes(context,
getIdentifierValue(tableName));
+ if (dataNodes.isEmpty()) {
+ return false;
+ }
+ String actualTableName = getIdentifierValue(tableName);
+ String actualIndexName = getIdentifierValue(indexName);
+
Awaitility.await().atMost(META_DATA_WAIT_TIMEOUT).pollInterval(META_DATA_POLL_INTERVAL)
+ .untilAsserted(() -> assertThat(String.format("Index `%s`
existed state mismatch", indexName), containsIndex(dataNodes, actualTableName,
actualIndexName), is(exists)));
+ return true;
+ }
+
+ private boolean containsIndex(final Collection<DataNode> dataNodes, final
String tableName, final String indexName) throws SQLException {
+ return getActualIndexes(dataNodes).stream().anyMatch(each ->
isSameIndex(each.getName(), tableName, indexName));
+ }
+
+ private boolean isSameIndex(final String actualIndexName, final String
tableName, final String indexName) {
+ if (null == actualIndexName) {
+ return false;
+ }
+ if (indexName.equalsIgnoreCase(actualIndexName)) {
+ return true;
+ }
+ String shardingIndexName = indexName + "_" + tableName;
+ return shardingIndexName.equalsIgnoreCase(actualIndexName) ||
actualIndexName.length() > shardingIndexName.length()
+ && '_' == actualIndexName.charAt(shardingIndexName.length())
&& actualIndexName.regionMatches(true, 0, shardingIndexName, 0,
shardingIndexName.length());
+ }
+
+ private Collection<DataNode> findDataNodes(final SQLE2EITContext context,
final String tableName) {
+ try {
+ return getDataNodes(context.getDataSet().findMetaData(tableName));
+ } catch (final IllegalArgumentException ignored) {
+ return new LinkedList<>();
+ }
+ }
+
+ private String getIdentifierValue(final String identifier) {
+ String result = identifier.trim();
+ if (result.contains(".")) {
+ result = result.substring(result.lastIndexOf('.') + 1);
+ }
+ return result.replace("`", "").replace("\"", "").replace("[",
"").replace("]", "");
+ }
+
+ private void assertTableMetaDataEventually(final AssertionTestParameter
testParam, final SQLE2EITContext context) {
+
Awaitility.await().atMost(META_DATA_WAIT_TIMEOUT).pollInterval(META_DATA_POLL_INTERVAL).untilAsserted(()
-> assertTableMetaData(testParam, context));
}
private void assertTableMetaData(final AssertionTestParameter testParam,
final SQLE2EITContext context) throws SQLException {
String tableName =
context.getAssertion().getInitialSQL().getAffectedTable();
DataSetMetaData expected =
context.getDataSet().findMetaData(tableName);
- Collection<DataNode> dataNodes =
InlineExpressionParserFactory.newInstance(expected.getDataNodes()).splitAndEvaluate().stream().map(DataNode::new).collect(Collectors.toList());
+ Collection<DataNode> dataNodes = getDataNodes(expected);
if (expected.getColumns().isEmpty()) {
assertNotContainsTable(environmentEngine, dataNodes);
return;
@@ -191,6 +303,10 @@ class DDLE2EIT implements SQLE2EIT {
assertIndexMetaData(actualIndexes, expected.getIndexes());
}
+ private Collection<DataNode> getDataNodes(final DataSetMetaData metaData) {
+ return
InlineExpressionParserFactory.newInstance(metaData.getDataNodes()).splitAndEvaluate().stream().map(DataNode::new).collect(Collectors.toList());
+ }
+
private void assertNotContainsTable(final SQLE2EEnvironmentEngine
environmentEngine, final Collection<DataNode> dataNodes) throws SQLException {
for (DataNode each : dataNodes) {
try (Connection connection =
environmentEngine.getActualDataSourceMap().get(each.getDataSourceName()).getConnection())
{
@@ -200,12 +316,16 @@ class DDLE2EIT implements SQLE2EIT {
}
private void assertNotContainsTable(final Connection connection, final
String tableName) throws SQLException {
- assertFalse(connection.getMetaData().getTables(null, null, tableName,
new String[]{"TABLE"}).next(), String.format("Table `%s` should not existed",
tableName));
+ assertFalse(containsTable(connection, tableName), String.format("Table
`%s` should not existed", tableName));
+ }
+
+ private boolean containsTable(final Connection connection, final String
tableName) throws SQLException {
+ return connection.getMetaData().getTables(null, null, tableName, new
String[]{"TABLE", "VIEW"}).next();
}
@SuppressWarnings("CollectionWithoutInitialCapacity")
private List<DataSetColumn> getActualColumns(final Collection<DataNode>
dataNodes) throws SQLException {
- Set<DataSetColumn> result = new LinkedHashSet<>();
+ Collection<DataSetColumn> result = new LinkedHashSet<>();
for (DataNode each : dataNodes) {
try (Connection connection =
environmentEngine.getActualDataSourceMap().get(each.getDataSourceName()).getConnection())
{
result.addAll(getActualColumns(connection,
each.getTableName()));
@@ -231,7 +351,7 @@ class DDLE2EIT implements SQLE2EIT {
@SuppressWarnings("CollectionWithoutInitialCapacity")
private List<DataSetIndex> getActualIndexes(final Collection<DataNode>
dataNodes) throws SQLException {
- Set<DataSetIndex> result = new LinkedHashSet<>();
+ Collection<DataSetIndex> result = new LinkedHashSet<>();
for (DataNode each : dataNodes) {
try (Connection connection =
environmentEngine.getActualDataSourceMap().get(each.getDataSourceName()).getConnection())
{
result.addAll(getActualIndexes(connection,
each.getTableName()));
@@ -290,7 +410,7 @@ class DDLE2EIT implements SQLE2EIT {
}
private void waitCompleted() {
- Awaitility.await().pollDelay(1500L, TimeUnit.MILLISECONDS).until(() ->
true);
+ Awaitility.await().pollDelay(DDL_COMPLETION_DELAY).until(() -> true);
}
private static boolean isEnabled() {
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BaseDMLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BaseDMLE2EIT.java
index c9369cf7ebb..d2cd78bd6f8 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BaseDMLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BaseDMLE2EIT.java
@@ -36,6 +36,7 @@ import
org.apache.shardingsphere.test.e2e.sql.cases.dataset.metadata.DataSetColu
import
org.apache.shardingsphere.test.e2e.sql.cases.dataset.metadata.DataSetMetaData;
import org.apache.shardingsphere.test.e2e.sql.cases.dataset.row.DataSetRow;
import org.apache.shardingsphere.test.e2e.sql.env.DataSetEnvironmentManager;
+import org.apache.shardingsphere.test.e2e.sql.env.DataSetResetScopeCalculator;
import org.apache.shardingsphere.test.e2e.sql.env.SQLE2EEnvironmentEngine;
import
org.apache.shardingsphere.test.e2e.sql.framework.metadata.DialectDatabaseAssertionMetaDataFactory;
import
org.apache.shardingsphere.test.e2e.sql.framework.metadata.DialectQueryBehaviorProvider;
@@ -81,8 +82,12 @@ public abstract class BaseDMLE2EIT implements SQLE2EIT {
private static final String DATA_COLUMN_DELIMITER = ", ";
+ private static final DataSetResetScopeCalculator
DATA_SET_RESET_SCOPE_CALCULATOR = new DataSetResetScopeCalculator();
+
private DataSetEnvironmentManager dataSetEnvironmentManager;
+ private Collection<String> resetTableNames = Collections.emptyList();
+
@Getter
@Setter
private SQLE2EEnvironmentEngine environmentEngine;
@@ -97,13 +102,18 @@ public abstract class BaseDMLE2EIT implements SQLE2EIT {
protected void init(final E2ETestParameter testParam) throws IOException,
JAXBException {
dataSetEnvironmentManager = new DataSetEnvironmentManager(
new ScenarioDataPath(testParam.getScenario(),
Type.ACTUAL).getDataSetFile(), getEnvironmentEngine().getActualDataSourceMap(),
testParam.getDatabaseType());
- dataSetEnvironmentManager.fillData();
+ resetTableNames =
DATA_SET_RESET_SCOPE_CALCULATOR.getResetTableNames(testParam);
+ fillActualDataSet(resetTableNames);
+ }
+
+ private void fillActualDataSet(final Collection<String> tableNames) {
+ dataSetEnvironmentManager.fillData(tableNames);
}
void tearDown() {
// TODO make sure test case can not be null
if (null != dataSetEnvironmentManager) {
- dataSetEnvironmentManager.cleanData();
+ dataSetEnvironmentManager.cleanData(resetTableNames);
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BatchDMLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BatchDMLE2EIT.java
index 8de207c9456..3fd1552c594 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BatchDMLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dml/BatchDMLE2EIT.java
@@ -106,7 +106,6 @@ class BatchDMLE2EIT extends BaseDMLE2EIT {
if (null == testParam.getTestCaseContext()) {
return;
}
- init(testParam);
try (
Connection connection =
getEnvironmentEngine().getTargetDataSource().getConnection();
PreparedStatement preparedStatement =
connection.prepareStatement(testParam.getTestCaseContext().getTestCase().getSql()))
{
@@ -115,8 +114,6 @@ class BatchDMLE2EIT extends BaseDMLE2EIT {
}
preparedStatement.clearBatch();
assertThat(preparedStatement.executeBatch().length, is(0));
- } finally {
- tearDown(testParam);
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/AdditionalDQLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/AdditionalDQLE2EIT.java
index 0e44981d0a3..b02b97ff190 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/AdditionalDQLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/AdditionalDQLE2EIT.java
@@ -26,6 +26,8 @@ import
org.apache.shardingsphere.test.e2e.sql.framework.type.SQLCommandType;
import org.apache.shardingsphere.test.e2e.sql.framework.type.SQLExecuteType;
import org.apache.shardingsphere.test.e2e.sql.it.SQLE2EITContext;
import org.junit.jupiter.api.condition.EnabledIf;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ArgumentsSource;
@@ -44,6 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class AdditionalDQLE2EIT extends BaseDQLE2EIT {
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecuteQueryWithResultSetTypeAndConcurrency(final
AssertionTestParameter testParam) throws SQLException, JAXBException,
IOException {
@@ -52,19 +55,11 @@ class AdditionalDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
- init(testParam, context);
- // TODO fix e2e test blocked exception with PostgreSQL or openGauss in
#23643
- if (isPostgreSQLOrOpenGauss(testParam.getDatabaseType().getType())) {
- return;
- }
- if (isUseXMLAsExpectedDataset()) {
- assertExecuteQueryWithXMLExpected(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- } else {
- assertExecuteQueryWithExpectedDataSource(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- }
+ executeDQL(context, () ->
assertExecuteQueryWithResultSetTypes(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY));
}
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecuteQueryWithResultSetTypeAndConcurrencyAndHoldability(final
AssertionTestParameter testParam) throws SQLException, JAXBException,
IOException {
@@ -73,20 +68,11 @@ class AdditionalDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
- init(testParam, context);
- // TODO fix e2e test blocked exception with PostgreSQL or openGauss in
#23643
- if (isPostgreSQLOrOpenGauss(testParam.getDatabaseType().getType())) {
- return;
- }
- if (isUseXMLAsExpectedDataset()) {
- assertExecuteQueryWithXMLExpected(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.HOLD_CURSORS_OVER_COMMIT);
- } else {
- assertExecuteQueryWithExpectedDataSource(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
- ResultSet.HOLD_CURSORS_OVER_COMMIT);
- }
+ executeDQL(context, () ->
assertExecuteQueryWithResultSetTypes(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.HOLD_CURSORS_OVER_COMMIT));
}
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecuteWithResultSetTypeAndConcurrency(final
AssertionTestParameter testParam) throws SQLException, JAXBException,
IOException {
@@ -95,19 +81,11 @@ class AdditionalDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
- init(testParam, context);
- // TODO fix e2e test blocked exception with PostgreSQL or openGauss in
#23643
- if (isPostgreSQLOrOpenGauss(testParam.getDatabaseType().getType())) {
- return;
- }
- if (isUseXMLAsExpectedDataset()) {
- assertExecuteWithXMLExpected(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- } else {
- assertExecuteWithExpectedDataSource(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- }
+ executeDQL(context, () -> assertExecuteWithResultSetTypes(testParam,
context, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY));
}
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecuteWithResultSetTypeAndConcurrencyAndHoldability(final
AssertionTestParameter testParam) throws SQLException, JAXBException,
IOException {
@@ -116,15 +94,33 @@ class AdditionalDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
+ executeDQL(context, () -> assertExecuteWithResultSetTypes(testParam,
context, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.HOLD_CURSORS_OVER_COMMIT));
+ }
+
+ private void assertExecuteQueryWithResultSetTypes(final
AssertionTestParameter testParam, final SQLE2EITContext context,
+ final int...
resultSetTypes) throws IOException, JAXBException, SQLException {
+ init(testParam, context);
+ // TODO fix e2e test blocked exception with PostgreSQL or openGauss in
#23643
+ if (isPostgreSQLOrOpenGauss(testParam.getDatabaseType().getType())) {
+ return;
+ }
+ if (isUseXMLAsExpectedDataset()) {
+ assertExecuteQueryWithXMLExpected(testParam, context,
resultSetTypes);
+ } else {
+ assertExecuteQueryWithExpectedDataSource(testParam, context,
resultSetTypes);
+ }
+ }
+
+ private void assertExecuteWithResultSetTypes(final AssertionTestParameter
testParam, final SQLE2EITContext context, final int... resultSetTypes) throws
IOException, JAXBException, SQLException {
init(testParam, context);
// TODO fix e2e test blocked exception with PostgreSQL or openGauss in
#23643
if (isPostgreSQLOrOpenGauss(testParam.getDatabaseType().getType())) {
return;
}
if (isUseXMLAsExpectedDataset()) {
- assertExecuteWithXMLExpected(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.HOLD_CURSORS_OVER_COMMIT);
+ assertExecuteWithXMLExpected(testParam, context, resultSetTypes);
} else {
- assertExecuteWithExpectedDataSource(testParam, context,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY,
ResultSet.HOLD_CURSORS_OVER_COMMIT);
+ assertExecuteWithExpectedDataSource(testParam, context,
resultSetTypes);
}
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/BaseDQLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/BaseDQLE2EIT.java
index c692a9c0507..3c2e7e7ace6 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/BaseDQLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/BaseDQLE2EIT.java
@@ -47,10 +47,15 @@ import java.sql.Types;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Collection;
-import java.util.HashSet;
+import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -61,7 +66,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Getter(AccessLevel.PROTECTED)
public abstract class BaseDQLE2EIT implements SQLE2EIT {
- private static final Collection<String> FILLED_SUITES = new HashSet<>();
+ private static final Collection<String> FILLED_SUITES =
ConcurrentHashMap.newKeySet();
+
+ private static final Map<String, Object> FILL_DATA_LOCKS = new
ConcurrentHashMap<>();
+
+ private static final ReadWriteLock DQL_EXECUTION_LOCK = new
ReentrantReadWriteLock(true);
private DataSource expectedDataSource;
@@ -70,6 +79,25 @@ public abstract class BaseDQLE2EIT implements SQLE2EIT {
@Setter
private SQLE2EEnvironmentEngine environmentEngine;
+ /**
+ * Execute DQL with concurrent execution control.
+ *
+ * @param context SQL E2E IT context
+ * @param callback DQL execution callback
+ * @throws SQLException SQL exception
+ * @throws IOException IO exception
+ * @throws JAXBException JAXB exception
+ */
+ protected final void executeDQL(final SQLE2EITContext context, final
DQLExecutionCallback callback) throws SQLException, IOException, JAXBException {
+ Lock lock =
DQLExclusiveExecutionDetector.requiresExclusiveExecution(context.getSQL()) ?
DQL_EXECUTION_LOCK.writeLock() : DQL_EXECUTION_LOCK.readLock();
+ lock.lock();
+ try {
+ callback.execute();
+ } finally {
+ lock.unlock();
+ }
+ }
+
protected final void init(final AssertionTestParameter testParam, final
SQLE2EITContext context) throws IOException, JAXBException {
fillDataOnlyOnce(testParam);
expectedDataSource = getExpectedDataSource(testParam, context);
@@ -92,16 +120,16 @@ public abstract class BaseDQLE2EIT implements SQLE2EIT {
if (FILLED_SUITES.contains(cacheKey)) {
return;
}
- synchronized (FILLED_SUITES) {
+ synchronized (FILL_DATA_LOCKS.computeIfAbsent(cacheKey, unused -> new
Object())) {
if (FILLED_SUITES.contains(cacheKey)) {
return;
}
new DataSetEnvironmentManager(
new ScenarioDataPath(testParam.getScenario(),
Type.ACTUAL).getDataSetFile(), getEnvironmentEngine().getActualDataSourceMap(),
- testParam.getDatabaseType()).fillData();
+
testParam.getDatabaseType()).fillData(Collections.emptyList());
new DataSetEnvironmentManager(
new ScenarioDataPath(testParam.getScenario(),
Type.EXPECTED).getDataSetFile(),
getEnvironmentEngine().getExpectedDataSourceMap(),
- testParam.getDatabaseType()).fillData();
+
testParam.getDatabaseType()).fillData(Collections.emptyList());
FILLED_SUITES.add(cacheKey);
}
}
@@ -229,4 +257,20 @@ public abstract class BaseDQLE2EIT implements SQLE2EIT {
assertThat(String.valueOf(actual.getObject(columnIndex)).trim(),
is(expected));
assertThat(String.valueOf(actual.getObject(columnLabel)).trim(),
is(expected));
}
+
+ /**
+ * DQL execution callback.
+ */
+ @FunctionalInterface
+ protected interface DQLExecutionCallback {
+
+ /**
+ * Execute DQL.
+ *
+ * @throws SQLException SQL exception
+ * @throws IOException IO exception
+ * @throws JAXBException JAXB exception
+ */
+ void execute() throws SQLException, IOException, JAXBException;
+ }
}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetector.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetector.java
new file mode 100644
index 00000000000..df1688267b1
--- /dev/null
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetector.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.test.e2e.sql.it.sql.dql;
+
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+/**
+ * Exclusive execution detector for SQL E2E tests.
+ */
+public final class DQLExclusiveExecutionDetector {
+
+ private static final Pattern SELECT_PATTERN =
Pattern.compile("\\bSELECT\\b");
+
+ private static final Pattern DML_RETURNING_PATTERN =
Pattern.compile("\\b(?:INSERT|UPDATE|DELETE|MERGE)\\b.*\\bRETURNING\\b");
+
+ private static final Pattern FOR_UPDATE_PATTERN =
Pattern.compile("\\bFOR\\s+(?:NO\\s+KEY\\s+)?UPDATE\\b");
+
+ private static final Pattern FOR_SHARE_PATTERN =
Pattern.compile("\\bFOR\\s+(?:KEY\\s+)?SHARE\\b");
+
+ private static final Pattern LOCK_IN_SHARE_MODE_PATTERN =
Pattern.compile("\\bLOCK\\s+IN\\s+SHARE\\s+MODE\\b");
+
+ private static final Pattern SQL_SERVER_LOCK_HINT_PATTERN =
Pattern.compile(
+
"\\bWITH\\s*\\([^)]*\\b(?:UPDLOCK|HOLDLOCK|XLOCK|TABLOCKX|TABLOCK|PAGLOCK|ROWLOCK|READCOMMITTEDLOCK|REPEATABLEREAD|SERIALIZABLE)\\b[^)]*\\)");
+
+ private static final Pattern KEEP_LOCKS_PATTERN =
Pattern.compile("\\bWITH\\s+(?:RR|RS|CS)\\s+USE\\s+AND\\s+KEEP\\s+(?:UPDATE|EXCLUSIVE|SHARE)\\s+LOCKS\\b");
+
+ private static final Pattern WITH_LOCK_PATTERN =
Pattern.compile("\\bWITH\\s+LOCK\\b(?!\\s+AS\\b)");
+
+ private DQLExclusiveExecutionDetector() {
+ }
+
+ /**
+ * Judge whether SQL requires exclusive execution.
+ *
+ * @param sql SQL
+ * @return requires exclusive execution or not
+ */
+ public static boolean requiresExclusiveExecution(final String sql) {
+ String normalizedSQL = normalize(sql);
+ return DML_RETURNING_PATTERN.matcher(normalizedSQL).find() ||
containsLockingSelect(normalizedSQL);
+ }
+
+ private static boolean containsLockingSelect(final String normalizedSQL) {
+ return SELECT_PATTERN.matcher(normalizedSQL).find()
+ && (FOR_UPDATE_PATTERN.matcher(normalizedSQL).find() ||
FOR_SHARE_PATTERN.matcher(normalizedSQL).find() ||
LOCK_IN_SHARE_MODE_PATTERN.matcher(normalizedSQL).find()
+ ||
SQL_SERVER_LOCK_HINT_PATTERN.matcher(normalizedSQL).find() ||
KEEP_LOCKS_PATTERN.matcher(normalizedSQL).find()
+ || WITH_LOCK_PATTERN.matcher(normalizedSQL).find());
+ }
+
+ private static String normalize(final String sql) {
+ StringBuilder result = new StringBuilder(sql.length());
+ int i = 0;
+ while (i < sql.length()) {
+ char each = sql.charAt(i);
+ if ('\'' == each || '"' == each || '`' == each) {
+ result.append(' ');
+ i = skipQuotedIdentifier(sql, i, each);
+ } else if ('[' == each) {
+ result.append(' ');
+ i = skipSquareQuotedIdentifier(sql, i);
+ } else if ('-' == each && i + 1 < sql.length() && '-' ==
sql.charAt(i + 1)) {
+ result.append(' ');
+ i = skipLineComment(sql, i + 2);
+ } else if ('#' == each) {
+ result.append(' ');
+ i = skipLineComment(sql, i + 1);
+ } else if ('/' == each && i + 1 < sql.length() && '*' ==
sql.charAt(i + 1)) {
+ result.append(' ');
+ i = skipBlockComment(sql, i);
+ } else {
+ result.append(each);
+ }
+ i++;
+ }
+ return
result.toString().toUpperCase(Locale.ENGLISH).replaceAll("\\s+", " ");
+ }
+
+ private static int skipQuotedIdentifier(final String sql, final int
startIndex, final char quoteCharacter) {
+ int i = startIndex + 1;
+ while (i < sql.length()) {
+ if ('\\' == sql.charAt(i)) {
+ i++;
+ } else if (quoteCharacter == sql.charAt(i)) {
+ if (i + 1 < sql.length() && quoteCharacter == sql.charAt(i +
1)) {
+ i++;
+ } else {
+ return i;
+ }
+ }
+ i++;
+ }
+ return sql.length() - 1;
+ }
+
+ private static int skipSquareQuotedIdentifier(final String sql, final int
startIndex) {
+ int i = startIndex + 1;
+ while (i < sql.length()) {
+ if (']' == sql.charAt(i)) {
+ return i;
+ }
+ i++;
+ }
+ return sql.length() - 1;
+ }
+
+ private static int skipLineComment(final String sql, final int startIndex)
{
+ int i = startIndex;
+ while (i < sql.length()) {
+ if ('\n' == sql.charAt(i)) {
+ return i;
+ }
+ i++;
+ }
+ return sql.length() - 1;
+ }
+
+ private static int skipBlockComment(final String sql, final int
startIndex) {
+ int i = startIndex + 2;
+ while (i < sql.length() - 1) {
+ if ('*' == sql.charAt(i) && '/' == sql.charAt(i + 1)) {
+ return i + 1;
+ }
+ i++;
+ }
+ return sql.length() - 1;
+ }
+}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetectorTest.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetectorTest.java
new file mode 100644
index 00000000000..34a82556244
--- /dev/null
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/DQLExclusiveExecutionDetectorTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.test.e2e.sql.it.sql.dql;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DQLExclusiveExecutionDetectorTest {
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("exclusiveExecutionSQLProvider")
+ void assertRequiresExclusiveExecution(final String name, final String sql)
{
+
assertTrue(DQLExclusiveExecutionDetector.requiresExclusiveExecution(sql));
+ }
+
+ private static Stream<Arguments> exclusiveExecutionSQLProvider() {
+ return Stream.of(
+ Arguments.of("For update", "SELECT * FROM t_order WHERE
order_id = 1 FOR UPDATE"),
+ Arguments.of("For no key update", "SELECT * FROM t_order FOR
NO KEY UPDATE"),
+ Arguments.of("For share", "SELECT * FROM t_order FOR SHARE"),
+ Arguments.of("For key share", "SELECT * FROM t_order FOR KEY
SHARE"),
+ Arguments.of("Lock in share mode", "SELECT * FROM t_order LOCK
IN SHARE MODE"),
+ Arguments.of("SQL Server updlock", "SELECT * FROM t_order WITH
(UPDLOCK) WHERE order_id = 1"),
+ Arguments.of("SQL Server holdlock", "SELECT * FROM t_order
WITH (ROWLOCK, HOLDLOCK)"),
+ Arguments.of("SQL Server xlock", "SELECT * FROM t_order WITH
(XLOCK, TABLOCKX)"),
+ Arguments.of("DB2 keep update locks", "SELECT * FROM t_order
WITH RS USE AND KEEP UPDATE LOCKS"),
+ Arguments.of("DB2 keep share locks", "SELECT * FROM t_order
WITH CS USE AND KEEP SHARE LOCKS"),
+ Arguments.of("With lock", "SELECT * FROM t_order WITH LOCK"),
+ Arguments.of("Insert returning", "INSERT INTO
t_with_generated_id (val) VALUES (?) RETURNING *, id, val aliased_val"),
+ Arguments.of("Update returning", "UPDATE t_order SET status =
'OK' WHERE order_id = 1 RETURNING order_id"),
+ Arguments.of("Delete returning", "DELETE FROM t_order WHERE
order_id = 1 RETURNING order_id"));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("nonExclusiveExecutionSQLProvider")
+ void assertDoesNotRequireExclusiveExecution(final String name, final
String sql) {
+
assertFalse(DQLExclusiveExecutionDetector.requiresExclusiveExecution(sql));
+ }
+
+ private static Stream<Arguments> nonExclusiveExecutionSQLProvider() {
+ return Stream.of(
+ Arguments.of("Plain select", "SELECT * FROM t_order"),
+ Arguments.of("For update literal", "SELECT 'FOR UPDATE' AS
content FROM t_order"),
+ Arguments.of("For update comment", "SELECT * FROM t_order --
FOR UPDATE"),
+ Arguments.of("For update block comment", "SELECT * FROM
t_order /* FOR UPDATE */"),
+ Arguments.of("For update quoted identifier", "SELECT \"FOR
UPDATE\" FROM t_order"),
+ Arguments.of("Non select DML", "UPDATE t_order SET status =
'FOR UPDATE' WHERE order_id = 1"),
+ Arguments.of("No lock hint", "SELECT * FROM t_order WITH
(NOLOCK)"),
+ Arguments.of("For read only", "SELECT * FROM t_order FOR READ
ONLY"),
+ Arguments.of("CTE named lock", "WITH lock AS (SELECT * FROM
t_order) SELECT * FROM lock"),
+ Arguments.of("Returning literal", "SELECT 'INSERT INTO t_order
RETURNING order_id' AS content FROM t_order"),
+ Arguments.of("Returning comment", "SELECT * FROM t_order --
INSERT INTO t_order RETURNING order_id"));
+ }
+}
diff --git
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/GeneralDQLE2EIT.java
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/GeneralDQLE2EIT.java
index df2fe7d5f0c..cbb60ed12b3 100644
---
a/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/GeneralDQLE2EIT.java
+++
b/test/e2e/sql/src/test/java/org/apache/shardingsphere/test/e2e/sql/it/sql/dql/GeneralDQLE2EIT.java
@@ -26,6 +26,8 @@ import
org.apache.shardingsphere.test.e2e.sql.framework.type.SQLCommandType;
import org.apache.shardingsphere.test.e2e.sql.framework.type.SQLExecuteType;
import org.apache.shardingsphere.test.e2e.sql.it.SQLE2EITContext;
import org.junit.jupiter.api.condition.EnabledIf;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ArgumentsSource;
@@ -43,6 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
class GeneralDQLE2EIT extends BaseDQLE2EIT {
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecuteQuery(final AssertionTestParameter testParam) throws
SQLException, IOException, JAXBException {
@@ -51,8 +54,10 @@ class GeneralDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
- init(testParam, context);
- assertExecuteQuery(testParam, context);
+ executeDQL(context, () -> {
+ init(testParam, context);
+ assertExecuteQuery(testParam, context);
+ });
}
private void assertExecuteQuery(final AssertionTestParameter testParam,
final SQLE2EITContext context) throws SQLException {
@@ -144,6 +149,7 @@ class GeneralDQLE2EIT extends BaseDQLE2EIT {
}
@ParameterizedTest(name = "{0}")
+ @Execution(ExecutionMode.CONCURRENT)
@EnabledIf("isEnabled")
@ArgumentsSource(SQLE2EITArgumentsProvider.class)
void assertExecute(final AssertionTestParameter testParam) throws
SQLException, JAXBException, IOException {
@@ -152,8 +158,10 @@ class GeneralDQLE2EIT extends BaseDQLE2EIT {
return;
}
SQLE2EITContext context = new SQLE2EITContext(testParam);
- init(testParam, context);
- assertExecute(testParam, context);
+ executeDQL(context, () -> {
+ init(testParam, context);
+ assertExecute(testParam, context);
+ });
}
private void assertExecute(final AssertionTestParameter testParam, final
SQLE2EITContext context) throws SQLException {
diff --git a/test/e2e/sql/src/test/resources/junit-platform.properties
b/test/e2e/sql/src/test/resources/junit-platform.properties
new file mode 100644
index 00000000000..5a10faf99c6
--- /dev/null
+++ b/test/e2e/sql/src/test/resources/junit-platform.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+junit.jupiter.execution.parallel.enabled=true
+junit.jupiter.execution.parallel.mode.default=same_thread
+junit.jupiter.execution.parallel.mode.classes.default=same_thread
+junit.jupiter.execution.parallel.config.strategy=fixed
+junit.jupiter.execution.parallel.config.fixed.parallelism=4
