This is an automated email from the ASF dual-hosted git repository.
panjuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new b532ad1 Refactor SingleTableDataCalculator dataCalculate method
parameter (#13340)
b532ad1 is described below
commit b532ad15ee4ff7822e4708694fa161a2793a559e
Author: Hongsheng Zhong <[email protected]>
AuthorDate: Fri Oct 29 09:54:53 2021 +0800
Refactor SingleTableDataCalculator dataCalculate method parameter (#13340)
---
.../scaling/core/api/DataCalculateParameter.java | 73 ++++++++++++++++++++++
.../core/api/SingleTableDataCalculator.java | 10 +--
...calingDefaultDataConsistencyCheckAlgorithm.java | 5 +-
.../consistency/DataConsistencyCheckerImpl.java | 7 ++-
.../impl/FixtureH2SingleTableDataCalculator.java | 6 +-
...ngDefaultDataConsistencyCheckAlgorithmTest.java | 4 +-
.../DefaultMySQLSingleTableDataCalculator.java | 10 +--
...DefaultPostgreSQLSingleTableDataCalculator.java | 6 +-
...owScalingCheckAlgorithmsQueryResultSetTest.java | 2 +-
9 files changed, 94 insertions(+), 29 deletions(-)
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/DataCalculateParameter.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/DataCalculateParameter.java
new file mode 100644
index 0000000..b98dcfb
--- /dev/null
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/DataCalculateParameter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.scaling.core.api;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import
org.apache.shardingsphere.scaling.core.config.datasource.ScalingDataSourceConfiguration;
+
+import java.util.Collection;
+
+/**
+ * Data calculate parameter.
+ */
+@Getter
+@Setter
+@Builder
+@ToString
+public final class DataCalculateParameter {
+
+ /**
+ * Data source configuration of source side or target side.
+ */
+ private ScalingDataSourceConfiguration dataSourceConfig;
+
+ private String logicTableName;
+
+ /**
+ * All column names of logic table.
+ */
+ private Collection<String> columnNames;
+
+ /**
+ * Peer database type.
+ */
+ private String peerDatabaseType;
+
+ /**
+ * Chunk size of limited records to be calculated in a batch.
+ */
+ private Integer chunkSize;
+
+ /**
+ * Ignored column names.
+ */
+ private Collection<String> ignoredColumnNames;
+
+ /**
+ * If {@link #chunkSize} exists, it could be used in order by clause on
first priority.
+ */
+ private Collection<String> primaryColumnNames;
+
+ /**
+ * If {@link #chunkSize} exists, it could be used in order by clause on
second priority.
+ */
+ private Collection<String> uniqueColumnNames;
+}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/SingleTableDataCalculator.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/SingleTableDataCalculator.java
index 030f419..24e39eb 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/SingleTableDataCalculator.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/SingleTableDataCalculator.java
@@ -17,10 +17,6 @@
package org.apache.shardingsphere.scaling.core.api;
-import
org.apache.shardingsphere.scaling.core.config.datasource.ScalingDataSourceConfiguration;
-
-import java.util.Collection;
-
/**
* Single table data calculator interface for SPI.
* <p>
@@ -46,10 +42,8 @@ public interface SingleTableDataCalculator {
/**
* Calculate table data, usually checksum.
*
- * @param dataSourceConfig data source configuration
- * @param logicTableName logic table name
- * @param columnNames column names
+ * @param dataCalculateParameter data calculate parameter
* @return calculated result, it will be used to check equality.
*/
- Object dataCalculate(ScalingDataSourceConfiguration dataSourceConfig,
String logicTableName, Collection<String> columnNames);
+ Object dataCalculate(DataCalculateParameter dataCalculateParameter);
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithm.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithm.java
index 317d70a..c4b6ece 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithm.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithm.java
@@ -18,12 +18,11 @@
package org.apache.shardingsphere.scaling.core.api.impl;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
-import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
import
org.apache.shardingsphere.scaling.core.api.ScalingDataConsistencyCheckAlgorithm;
import org.apache.shardingsphere.scaling.core.api.SingleTableDataCalculator;
-import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
/**
* Scaling default data consistency check algorithm.
@@ -32,7 +31,7 @@ public final class
ScalingDefaultDataConsistencyCheckAlgorithm implements Scalin
public static final String TYPE = "DEFAULT";
- private static final Collection<String> SUPPORTED_DATABASE_TYPES =
Arrays.asList(new MySQLDatabaseType().getName(), new
PostgreSQLDatabaseType().getName());
+ private static final Collection<String> SUPPORTED_DATABASE_TYPES =
Collections.singletonList(new MySQLDatabaseType().getName());
@Override
public void init() {
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/job/check/consistency/DataConsistencyCheckerImpl.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/job/check/consistency/DataConsistencyCheckerImpl.java
index a77d993..5c53005 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/job/check/consistency/DataConsistencyCheckerImpl.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/main/java/org/apache/shardingsphere/scaling/core/job/check/consistency/DataConsistencyCheckerImpl.java
@@ -22,6 +22,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
import
org.apache.shardingsphere.infra.executor.kernel.thread.ExecutorThreadFactoryBuilder;
+import org.apache.shardingsphere.scaling.core.api.DataCalculateParameter;
import
org.apache.shardingsphere.scaling.core.api.ScalingDataConsistencyCheckAlgorithm;
import org.apache.shardingsphere.scaling.core.api.SingleTableDataCalculator;
import
org.apache.shardingsphere.scaling.core.common.datasource.DataSourceFactory;
@@ -127,8 +128,10 @@ public final class DataConsistencyCheckerImpl implements
DataConsistencyChecker
try {
for (String each : logicTableNames) {
Collection<String> columnNames =
tablesColumnNamesMap.get(each);
- Future<Object> sourceFuture = executor.submit(() ->
sourceCalculator.dataCalculate(sourceConfig, each, columnNames));
- Future<Object> targetFuture = executor.submit(() ->
targetCalculator.dataCalculate(targetConfig, each, columnNames));
+ DataCalculateParameter sourceCalculateParameter =
DataCalculateParameter.builder().dataSourceConfig(sourceConfig).logicTableName(each).columnNames(columnNames).build();
+ Future<Object> sourceFuture = executor.submit(() ->
sourceCalculator.dataCalculate(sourceCalculateParameter));
+ DataCalculateParameter targetCalculateParameter =
DataCalculateParameter.builder().dataSourceConfig(targetConfig).logicTableName(each).columnNames(columnNames).build();
+ Future<Object> targetFuture = executor.submit(() ->
targetCalculator.dataCalculate(targetCalculateParameter));
Object sourceCalculateResult = sourceFuture.get();
Object targetCalculateResult = targetFuture.get();
boolean calculateResultsEquals =
Objects.equals(sourceCalculateResult, targetCalculateResult);
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/FixtureH2SingleTableDataCalculator.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/FixtureH2SingleTableDataCalculator.java
index cb2543a..1292d25 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/FixtureH2SingleTableDataCalculator.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/FixtureH2SingleTableDataCalculator.java
@@ -17,10 +17,8 @@
package org.apache.shardingsphere.scaling.core.api.impl;
+import org.apache.shardingsphere.scaling.core.api.DataCalculateParameter;
import org.apache.shardingsphere.scaling.core.api.SingleTableDataCalculator;
-import
org.apache.shardingsphere.scaling.core.config.datasource.ScalingDataSourceConfiguration;
-
-import java.util.Collection;
public final class FixtureH2SingleTableDataCalculator implements
SingleTableDataCalculator {
@@ -35,7 +33,7 @@ public final class FixtureH2SingleTableDataCalculator
implements SingleTableData
}
@Override
- public Object dataCalculate(final ScalingDataSourceConfiguration
dataSourceConfig, final String logicTableName, final Collection<String>
columnNames) {
+ public Object dataCalculate(final DataCalculateParameter
dataCalculateParameter) {
return true;
}
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithmTest.java
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithmTest.java
index 19eebb8..5c9b001 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithmTest.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-core/src/test/java/org/apache/shardingsphere/scaling/core/api/impl/ScalingDefaultDataConsistencyCheckAlgorithmTest.java
@@ -19,7 +19,7 @@ package org.apache.shardingsphere.scaling.core.api.impl;
import org.junit.Test;
-import java.util.Arrays;
+import java.util.Collections;
import static org.hamcrest.CoreMatchers.is;
import static org.junit.Assert.assertNotNull;
@@ -34,7 +34,7 @@ public final class
ScalingDefaultDataConsistencyCheckAlgorithmTest {
assertThat(checkAlgorithm.getType(),
is(ScalingDefaultDataConsistencyCheckAlgorithm.TYPE));
assertNotNull(checkAlgorithm.getDescription());
assertThat(checkAlgorithm.getProvider(), is("ShardingSphere"));
- assertThat(checkAlgorithm.getSupportedDatabaseTypes(),
is(Arrays.asList("MySQL", "PostgreSQL")));
+ assertThat(checkAlgorithm.getSupportedDatabaseTypes(),
is(Collections.singletonList("MySQL")));
}
@Test(expected = NullPointerException.class)
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-mysql/src/main/java/org/apache/shardingsphere/scaling/mysql/component/checker/DefaultMySQLSingleTableDataCalculator.java
b/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-mysql/src/main/java/org/apache/shardingsphere/scaling/mysql/component/checker/DefaultMySQLSingleTableDataCalculator.java
index e1de3c0..16543f0 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-mysql/src/main/java/org/apache/shardingsphere/scaling/mysql/component/checker/DefaultMySQLSingleTableDataCalculator.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-mysql/src/main/java/org/apache/shardingsphere/scaling/mysql/component/checker/DefaultMySQLSingleTableDataCalculator.java
@@ -18,11 +18,11 @@
package org.apache.shardingsphere.scaling.mysql.component.checker;
import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
+import org.apache.shardingsphere.scaling.core.api.DataCalculateParameter;
import
org.apache.shardingsphere.scaling.core.api.impl.AbstractSingleTableDataCalculator;
import
org.apache.shardingsphere.scaling.core.api.impl.ScalingDefaultDataConsistencyCheckAlgorithm;
import
org.apache.shardingsphere.scaling.core.common.datasource.DataSourceWrapper;
import
org.apache.shardingsphere.scaling.core.common.exception.DataCheckFailException;
-import
org.apache.shardingsphere.scaling.core.config.datasource.ScalingDataSourceConfiguration;
import
org.apache.shardingsphere.scaling.mysql.component.MySQLScalingSQLBuilder;
import javax.sql.DataSource;
@@ -30,7 +30,6 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.Collection;
import java.util.HashMap;
import java.util.stream.Collectors;
@@ -52,10 +51,11 @@ public final class DefaultMySQLSingleTableDataCalculator
extends AbstractSingleT
}
@Override
- public Object dataCalculate(final ScalingDataSourceConfiguration
dataSourceConfig, final String logicTableName, final Collection<String>
columnNames) {
+ public Object dataCalculate(final DataCalculateParameter
dataCalculateParameter) {
+ String logicTableName = dataCalculateParameter.getLogicTableName();
MySQLScalingSQLBuilder scalingSQLBuilder = new
MySQLScalingSQLBuilder(new HashMap<>());
- try (DataSourceWrapper dataSource = getDataSource(dataSourceConfig)) {
- return columnNames.stream().map(each -> {
+ try (DataSourceWrapper dataSource =
getDataSource(dataCalculateParameter.getDataSourceConfig())) {
+ return dataCalculateParameter.getColumnNames().stream().map(each
-> {
String sql =
scalingSQLBuilder.buildSumCrc32SQL(logicTableName, each);
return sumCrc32(dataSource, sql);
}).collect(Collectors.toList());
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-postgresql/src/main/java/org/apache/shardingsphere/scaling/postgresql/component/checker/DefaultPostgreSQLSingleTableDataCalculator.java
b/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-postgresql/src/main/java/org/apache/shardingsphere/scaling/postgresql/component/checker/DefaultPostgreSQLSingleTableDataCalculator.java
index f87ed19..42b37f8 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-postgresql/src/main/java/org/apache/shardingsphere/scaling/postgresql/component/checker/DefaultPostgreSQLSingleTableDataCalculator.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-dialect/shardingsphere-scaling-postgresql/src/main/java/org/apache/shardingsphere/scaling/postgresql/component/checker/DefaultPostgreSQLSingleTableDataCalculator.java
@@ -18,11 +18,9 @@
package org.apache.shardingsphere.scaling.postgresql.component.checker;
import
org.apache.shardingsphere.infra.database.type.dialect.PostgreSQLDatabaseType;
+import org.apache.shardingsphere.scaling.core.api.DataCalculateParameter;
import
org.apache.shardingsphere.scaling.core.api.impl.AbstractSingleTableDataCalculator;
import
org.apache.shardingsphere.scaling.core.api.impl.ScalingDefaultDataConsistencyCheckAlgorithm;
-import
org.apache.shardingsphere.scaling.core.config.datasource.ScalingDataSourceConfiguration;
-
-import java.util.Collection;
/**
* Default PostgreSQL single table data calculator.
@@ -42,7 +40,7 @@ public final class DefaultPostgreSQLSingleTableDataCalculator
extends AbstractSi
}
@Override
- public Object dataCalculate(final ScalingDataSourceConfiguration
dataSourceConfig, final String logicTableName, final Collection<String>
columnNames) {
+ public Object dataCalculate(final DataCalculateParameter
dataCalculateParameter) {
//TODO PostgreSQL dataCalculate
return true;
}
diff --git
a/shardingsphere-scaling/shardingsphere-scaling-distsql/shardingsphere-scaling-distsql-handler/src/test/java/org/apache/shardingsphere/scaling/distsql/handler/ShowScalingCheckAlgorithmsQueryResultSetTest.java
b/shardingsphere-scaling/shardingsphere-scaling-distsql/shardingsphere-scaling-distsql-handler/src/test/java/org/apache/shardingsphere/scaling/distsql/handler/ShowScalingCheckAlgorithmsQueryResultSetTest.java
index 8385b10..8e8e3c6 100644
---
a/shardingsphere-scaling/shardingsphere-scaling-distsql/shardingsphere-scaling-distsql-handler/src/test/java/org/apache/shardingsphere/scaling/distsql/handler/ShowScalingCheckAlgorithmsQueryResultSetTest.java
+++
b/shardingsphere-scaling/shardingsphere-scaling-distsql/shardingsphere-scaling-distsql-handler/src/test/java/org/apache/shardingsphere/scaling/distsql/handler/ShowScalingCheckAlgorithmsQueryResultSetTest.java
@@ -65,7 +65,7 @@ public final class
ShowScalingCheckAlgorithmsQueryResultSetTest {
Iterator<Object> rowData = actual.iterator();
assertThat(rowData.next(), is("DEFAULT"));
assertThat(rowData.next(), is("Default implementation with CRC32 of
all records."));
- assertThat(rowData.next(), is("MySQL,PostgreSQL"));
+ assertThat(rowData.next(), is("MySQL"));
assertThat(rowData.next(), is("ShardingSphere"));
}
}
