This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 09a50a0c7c6 Run scaling IT test for different DBs one by one (#18387)
09a50a0c7c6 is described below
commit 09a50a0c7c6bae15745245464d47e5953b68cc18
Author: azexcy <[email protected]>
AuthorDate: Thu Jun 16 19:03:15 2022 +0800
Run scaling IT test for different DBs one by one (#18387)
---
.github/workflows/it-scaling.yml | 14 ++++---
.../data/pipeline/cases/base/BaseITCase.java | 44 ++++++++++------------
.../cases/general/MySQLGeneralScalingIT.java | 9 ++++-
.../cases/general/PostgreSQLGeneralScalingIT.java | 11 +++++-
.../cases/primarykey/TextPrimaryKeyScalingIT.java | 7 +++-
.../src/test/resources/env/logback.xml | 3 --
6 files changed, 52 insertions(+), 36 deletions(-)
diff --git a/.github/workflows/it-scaling.yml b/.github/workflows/it-scaling.yml
index 75d842d1916..12c5c33d0d6 100644
--- a/.github/workflows/it-scaling.yml
+++ b/.github/workflows/it-scaling.yml
@@ -86,9 +86,11 @@ jobs:
java-version: 8
- name: Build Scaling IT image
run: ./mvnw -B clean install -am -pl
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling
-Pit.env.docker -DskipTests
- - name: Run Scaling Integration Test
- run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.mysql.version=${{ env.mysql_version }}
-Dit.env.postgresql.version=${{ env.postgresql_version }}
-
+ - name: Run Scaling MySQL Integration Test
+ run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.mysql.version=${{ env.mysql_version }}
+ - name: Run Scaling PostgreSQL Integration Test
+ run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.postgresql.version=${{
env.postgresql_version }}
+
scaling-daily-it-test:
if: (github.event_name == 'schedule' && github.repository ==
'apache/shardingsphere')
name: scaling-daily-it-test
@@ -112,5 +114,7 @@ jobs:
java-version: 8
- name: Build Scaling IT image
run: ./mvnw -B clean install -am -pl
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling
-Pit.env.docker -DskipTests
- - name: Run Scaling Daily Integration Test
- run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.postgresql.version=${{
env.postgresql_version }} -Dit.env.mysql.version=${{ env.mysql_version }}
+ - name: Run Scaling Daily MySQL Integration Test
+ run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.mysql.version=${{ env.mysql_version }}
+ - name: Run Scaling Daily PostgreSQL Integration Test
+ run: ./mvnw -nsu -B install -f
shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/pom.xml
-Dit.cluster.env.type=DOCKER -Dit.env.postgresql.version=${{
env.postgresql_version }}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
index 2fc0962774e..dd43262ee17 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/base/BaseITCase.java
@@ -57,7 +57,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -128,7 +127,7 @@ public abstract class BaseITCase {
long startTime = System.currentTimeMillis();
int waitTimes = 0;
do {
- List<Map<String, Object>> result = jdbcTemplate.queryForList("SHOW
SHARDING ALGORITHMS");
+ List<Map<String, Object>> result = queryForListWithLog("SHOW
SHARDING ALGORITHMS");
if (result.size() >= 3) {
log.info("waitShardingAlgorithmEffect time consume: {}",
System.currentTimeMillis() - startTime);
return true;
@@ -261,21 +260,20 @@ public abstract class BaseITCase {
previewResults.stream().map(each ->
each.get("data_source_name")).collect(Collectors.toSet()), is(new
HashSet<>(Arrays.asList("ds_0", "ds_1"))));
}
- /**
- * Check data match consistency.
- *
- * @throws InterruptedException interrupted exception
- */
- protected void assertCheckMatchConsistencySuccess() throws
InterruptedException {
+ protected String getScalingJobId() {
assertBeforeApplyScalingMetadataCorrectly();
- if (null != increaseTaskThread) {
- increaseTaskThread.join(60 * 1000L);
- }
- TimeUnit.SECONDS.sleep(4);
List<Map<String, Object>> scalingListMap = queryForListWithLog("SHOW
SCALING LIST");
assertThat(scalingListMap.size(), is(1));
String jobId = scalingListMap.get(0).get("id").toString();
log.info("jobId: {}", jobId);
+ return jobId;
+ }
+
+ protected void waitScalingFinished(final String jobId) throws
InterruptedException {
+ if (null != increaseTaskThread) {
+ TimeUnit.SECONDS.timedJoin(increaseTaskThread, 60);
+ }
+ log.info("jobId: {}", jobId);
Map<String, String> actualStatusMap = new HashMap<>(2, 1);
String showScalingStatus = String.format("SHOW SCALING STATUS %s",
jobId);
for (int i = 0; i < 15; i++) {
@@ -301,27 +299,25 @@ public abstract class BaseITCase {
TimeUnit.SECONDS.sleep(2);
}
assertThat(actualStatusMap.values().stream().filter(StringUtils::isNotBlank).collect(Collectors.toSet()).size(),
is(1));
+ }
+
+ protected void assertCheckScalingSuccess(final String jobId) {
stopScalingSourceWriting(jobId);
assertStopScalingSourceWriting();
- assertBeforeApplyScalingMetadataCorrectly();
List<Map<String, Object>> checkScalingResults =
queryForListWithLog(String.format("CHECK SCALING %s BY TYPE (NAME=DATA_MATCH)",
jobId));
log.info("checkScalingResults: {}", checkScalingResults);
for (Map<String, Object> entry : checkScalingResults) {
assertTrue(Boolean.parseBoolean(entry.get("records_content_matched").toString()));
}
- assertBeforeApplyScalingMetadataCorrectly();
- applyScaling(jobId);
- // TODO make sure the scaling job was applied
- TimeUnit.SECONDS.sleep(2);
- List<Map<String, Object>> previewResults =
queryForListWithLog("PREVIEW SELECT COUNT(1) FROM t_order");
- Set<Object> targetSources = previewResults.stream().map(each ->
each.get("actual_sql")).collect(Collectors.toSet());
- assertThat(previewResults.stream().map(each ->
each.get("data_source_name")).collect(Collectors.toSet()), is(new
HashSet<>(Arrays.asList("ds_2", "ds_3", "ds_4"))));
- assertThat(targetSources, is(new HashSet<>(Arrays.asList("SELECT
COUNT(1) FROM t_order_0 UNION ALL SELECT COUNT(1) FROM t_order_3",
- "SELECT COUNT(1) FROM t_order_1 UNION ALL SELECT COUNT(1) FROM
t_order_4", "SELECT COUNT(1) FROM t_order_2 UNION ALL SELECT COUNT(1) FROM
t_order_5"))));
- restoreScalingSourceWriting(jobId);
}
- private void restoreScalingSourceWriting(final String jobId) {
+ protected void assertPreviewTableSuccess(final String tableName, final
List<String> expect) {
+ List<Map<String, Object>> actualResults =
queryForListWithLog(String.format("PREVIEW SELECT COUNT(1) FROM %s",
tableName));
+ List<String> dataSourceNames = actualResults.stream().map(each ->
String.valueOf(each.get("data_source_name"))).collect(Collectors.toList());
+ assertThat(dataSourceNames, is(expect));
+ }
+
+ protected void restoreScalingSourceWriting(final String jobId) {
executeWithLog(String.format("RESTORE SCALING SOURCE WRITING %s",
jobId));
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
index 861a21879bf..82450c25152 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/MySQLGeneralScalingIT.java
@@ -31,6 +31,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -82,7 +83,13 @@ public final class MySQLGeneralScalingIT extends
BaseExtraSQLITCase {
startIncrementTask(new MySQLIncrementTask(getJdbcTemplate(),
keyGenerateAlgorithm, true));
addTargetResource();
executeWithLog(getCommonSQLCommand().getAutoAlterOrderWithItemShardingTableRule());
- assertCheckMatchConsistencySuccess();
+ String jobId = getScalingJobId();
+ waitScalingFinished(jobId);
+ assertCheckScalingSuccess(jobId);
+ applyScaling(jobId);
+ assertPreviewTableSuccess("t_order", Arrays.asList("ds_2", "ds_3",
"ds_4"));
+ assertPreviewTableSuccess("t_order_item", Arrays.asList("ds_2",
"ds_3", "ds_4"));
+ restoreScalingSourceWriting(jobId);
assertRestoreScalingSourceWriting();
}
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
index 8dd0c73e8e3..2bdc5fa6b4a 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/general/PostgreSQLGeneralScalingIT.java
@@ -32,6 +32,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -84,8 +85,14 @@ public final class PostgreSQLGeneralScalingIT extends
BaseExtraSQLITCase {
getJdbcTemplate().batchUpdate(getExtraSQLCommand().getFullInsertOrderItem(),
dataPair.getRight());
startIncrementTask(new PostgreSQLIncrementTask(getJdbcTemplate(), new
SnowflakeKeyGenerateAlgorithm(), "test", true));
addTargetResource();
-
getJdbcTemplate().execute(getCommonSQLCommand().getAutoAlterOrderWithItemShardingTableRule());
- assertCheckMatchConsistencySuccess();
+
executeWithLog(getCommonSQLCommand().getAutoAlterOrderWithItemShardingTableRule());
+ String jobId = getScalingJobId();
+ waitScalingFinished(jobId);
+ assertCheckScalingSuccess(jobId);
+ applyScaling(jobId);
+ assertPreviewTableSuccess("t_order", Arrays.asList("ds_2", "ds_3",
"ds_4"));
+ assertPreviewTableSuccess("t_order_item", Arrays.asList("ds_2",
"ds_3", "ds_4"));
+ restoreScalingSourceWriting(jobId);
assertRestoreScalingSourceWriting();
}
}
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
index e7f5979d31c..318e764c8e2 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/java/org/apache/shardingsphere/integration/data/pipeline/cases/primarykey/TextPrimaryKeyScalingIT.java
@@ -31,6 +31,7 @@ import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
@@ -75,7 +76,11 @@ public class TextPrimaryKeyScalingIT extends
BaseExtraSQLITCase {
batchInsertOrder();
addTargetResource();
executeWithLog(getCommonSQLCommand().getAutoAlterOrderShardingTableRule());
- assertCheckMatchConsistencySuccess();
+ String jobId = getScalingJobId();
+ waitScalingFinished(jobId);
+ assertCheckScalingSuccess(jobId);
+ applyScaling(jobId);
+ assertPreviewTableSuccess("t_order", Arrays.asList("ds_2", "ds_3",
"ds_4"));
}
private void batchInsertOrder() {
diff --git
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/logback.xml
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/logback.xml
index 487879aca84..7597a483675 100644
---
a/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/logback.xml
+++
b/shardingsphere-test/shardingsphere-integration-test/shardingsphere-integration-test-scaling/src/test/resources/env/logback.xml
@@ -31,9 +31,6 @@
<logger name="org.apache.shardingsphere.driver" level="INFO"
additivity="false">
<appender-ref ref="console" />
</logger>
- <logger
name="org.apache.shardingsphere.mode.manager.cluster.coordinator.lock"
level="DEBUG" additivity="false">
- <appender-ref ref="console" />
- </logger>
<logger name="com.zaxxer.hikari.pool.ProxyConnection" level="OFF" />
<root>
<level value="WARN" />
