This is an automated email from the ASF dual-hosted git repository.
azexin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 8969287567f Refactor DataMatchCalculatedResult (#25861)
8969287567f is described below
commit 8969287567faa9c796eb135c458719fc24eadf2f
Author: Hongsheng Zhong <[email protected]>
AuthorDate: Tue May 23 16:22:42 2023 +0800
Refactor DataMatchCalculatedResult (#25861)
* Rename CalculatedResult to DataMatchCalculatedResult
* Extract DataMatchCalculatedResult as dividual class
* Refactor DataMatchCalculatedResult
* Extract recordsEquals method to DataConsistencyCheckUtils
---
.../consistency/DataConsistencyCheckUtils.java | 45 ++++++++++
.../consistency/DataMatchCalculatedResult.java | 97 ++++++++++++++++++++
...RC32MatchDataConsistencyCalculateAlgorithm.java | 2 +-
...DataMatchDataConsistencyCalculateAlgorithm.java | 100 +--------------------
.../DataMatchCalculatedResultTest.java | 27 +++---
5 files changed, 160 insertions(+), 111 deletions(-)
diff --git
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckUtils.java
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckUtils.java
index c79f4c3c73b..6ac7331ccb3 100644
---
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckUtils.java
+++
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckUtils.java
@@ -19,16 +19,61 @@ package
org.apache.shardingsphere.data.pipeline.core.check.consistency;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.builder.EqualsBuilder;
import java.math.BigDecimal;
import java.math.RoundingMode;
+import java.sql.Array;
+import java.sql.SQLException;
+import java.sql.SQLXML;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Objects;
/**
* Data consistency check utility class.
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
+@Slf4j
public final class DataConsistencyCheckUtils {
+ /**
+ * Whether records equals or not.
+ *
+ * @param thisRecord this record
+ * @param thatRecord that record
+ * @param equalsBuilder equals builder
+ * @return true if records equals, otherwise false
+ * @throws SQLException if getting special value failed
+ */
+ public static boolean recordsEquals(final Collection<Object> thisRecord,
final Collection<Object> thatRecord, final EqualsBuilder equalsBuilder) throws
SQLException {
+ Iterator<Object> thisRecordIterator = thisRecord.iterator();
+ Iterator<Object> thatRecordIterator = thatRecord.iterator();
+ int columnIndex = 0;
+ while (thisRecordIterator.hasNext() && thatRecordIterator.hasNext()) {
+ ++columnIndex;
+ Object thisColumnValue = thisRecordIterator.next();
+ Object thatColumnValue = thatRecordIterator.next();
+ boolean matched;
+ if (thisColumnValue instanceof SQLXML && thatColumnValue
instanceof SQLXML) {
+ matched = ((SQLXML)
thisColumnValue).getString().equals(((SQLXML) thatColumnValue).getString());
+ } else if (thisColumnValue instanceof BigDecimal &&
thatColumnValue instanceof BigDecimal) {
+ matched = isBigDecimalEquals((BigDecimal) thisColumnValue,
(BigDecimal) thatColumnValue);
+ } else if (thisColumnValue instanceof Array && thatColumnValue
instanceof Array) {
+ matched = Objects.deepEquals(((Array)
thisColumnValue).getArray(), ((Array) thatColumnValue).getArray());
+ } else {
+ matched = equalsBuilder.append(thisColumnValue,
thatColumnValue).isEquals();
+ }
+ if (!matched) {
+ log.warn("Record column value not match, columnIndex={},
value1={}, value2={}, value1.class={}, value2.class={}.", columnIndex,
thisColumnValue, thatColumnValue,
+ null != thisColumnValue ?
thisColumnValue.getClass().getName() : "", null != thatColumnValue ?
thatColumnValue.getClass().getName() : "");
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Check two BigDecimal whether equals or not.
*
diff --git
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResult.java
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResult.java
new file mode 100644
index 00000000000..a916300e9f8
--- /dev/null
+++
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResult.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.data.pipeline.core.check.consistency;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+import
org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCalculatedResult;
+
+import java.sql.SQLException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Data match calculated result.
+ */
+@RequiredArgsConstructor
+@Getter
+@Slf4j
+public final class DataMatchCalculatedResult implements
DataConsistencyCalculatedResult {
+
+ @NonNull
+ private final Object maxUniqueKeyValue;
+
+ private final int recordsCount;
+
+ private final Collection<Collection<Object>> records;
+
+ @Override
+ public Optional<Object> getMaxUniqueKeyValue() {
+ return Optional.of(maxUniqueKeyValue);
+ }
+
+ @SneakyThrows(SQLException.class)
+ @Override
+ public boolean equals(final Object o) {
+ if (null == o) {
+ return false;
+ }
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof DataMatchCalculatedResult)) {
+ log.warn("DataMatchCalculatedResult type not match,
o.className={}.", o.getClass().getName());
+ return false;
+ }
+ final DataMatchCalculatedResult that = (DataMatchCalculatedResult) o;
+ if (recordsCount != that.recordsCount ||
!Objects.equals(maxUniqueKeyValue, that.maxUniqueKeyValue)) {
+ log.warn("Record count or max unique key value not match,
recordCount1={}, recordCount2={}, maxUniqueKeyValue1={},
maxUniqueKeyValue2={}.",
+ recordsCount, that.recordsCount, maxUniqueKeyValue,
that.maxUniqueKeyValue);
+ return false;
+ }
+ EqualsBuilder equalsBuilder = new EqualsBuilder();
+ Iterator<Collection<Object>> thisRecordsIterator = records.iterator();
+ Iterator<Collection<Object>> thatRecordsIterator =
that.records.iterator();
+ while (thisRecordsIterator.hasNext() && thatRecordsIterator.hasNext())
{
+ equalsBuilder.reset();
+ Collection<Object> thisRecord = thisRecordsIterator.next();
+ Collection<Object> thatRecord = thatRecordsIterator.next();
+ if (thisRecord.size() != thatRecord.size()) {
+ log.warn("Record column size not match, size1={}, size2={},
record1={}, record2={}.", thisRecord.size(), thatRecord.size(), thisRecord,
thatRecord);
+ return false;
+ }
+ if (!DataConsistencyCheckUtils.recordsEquals(thisRecord,
thatRecord, equalsBuilder)) {
+ log.warn("Records not equals, record1={}, record2={}.",
thatRecord, thatRecord);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder(17,
37).append(getMaxUniqueKeyValue().orElse(null)).append(getRecordsCount()).append(getRecords()).toHashCode();
+ }
+}
diff --git
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
index dc4bc095f09..90b2e1b2d0a 100644
---
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
+++
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/CRC32MatchDataConsistencyCalculateAlgorithm.java
@@ -110,7 +110,7 @@ public final class
CRC32MatchDataConsistencyCalculateAlgorithm extends AbstractD
return true;
}
if (getClass() != o.getClass()) {
- log.warn("CalculatedResult type not match, o.className={}",
o.getClass().getName());
+ log.warn("DataMatchCalculatedResult type not match,
o.className={}", o.getClass().getName());
return false;
}
final CalculatedResult that = (CalculatedResult) o;
diff --git
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
index 75f71ddff57..eb48732ce45 100644
---
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
+++
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchDataConsistencyCalculateAlgorithm.java
@@ -18,15 +18,11 @@
package
org.apache.shardingsphere.data.pipeline.core.check.consistency.algorithm;
import lombok.Getter;
-import lombok.NonNull;
import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
import
org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCalculateParameter;
import
org.apache.shardingsphere.data.pipeline.api.check.consistency.DataConsistencyCalculatedResult;
-import
org.apache.shardingsphere.data.pipeline.core.check.consistency.DataConsistencyCheckUtils;
+import
org.apache.shardingsphere.data.pipeline.core.check.consistency.DataMatchCalculatedResult;
import
org.apache.shardingsphere.data.pipeline.core.exception.PipelineSQLException;
import
org.apache.shardingsphere.data.pipeline.core.exception.data.PipelineTableDataConsistencyCheckLoadingFailedException;
import org.apache.shardingsphere.data.pipeline.core.util.CloseUtils;
@@ -41,18 +37,13 @@ import
org.apache.shardingsphere.infra.util.spi.ShardingSphereServiceLoader;
import org.apache.shardingsphere.infra.util.spi.annotation.SPIDescription;
import org.apache.shardingsphere.infra.util.spi.type.typed.TypedSPILoader;
-import java.math.BigDecimal;
-import java.sql.Array;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
-import java.sql.SQLXML;
import java.util.Collection;
-import java.util.Iterator;
import java.util.LinkedList;
-import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicReference;
@@ -119,7 +110,7 @@ public final class
DataMatchDataConsistencyCalculateAlgorithm extends AbstractSt
if (records.isEmpty()) {
calculationContext.close();
}
- return records.isEmpty() ? Optional.empty() : Optional.of(new
CalculatedResult(maxUniqueKeyValue, records.size(), records));
+ return records.isEmpty() ? Optional.empty() : Optional.of(new
DataMatchCalculatedResult(maxUniqueKeyValue, records.size(), records));
} catch (final PipelineSQLException ex) {
calculationContext.close();
throw ex;
@@ -212,7 +203,7 @@ public final class
DataMatchDataConsistencyCalculateAlgorithm extends AbstractSt
/**
* Set prepared statement.
- *
+ *
* @param preparedStatement prepared statement
*/
public void setPreparedStatement(final PreparedStatement
preparedStatement) {
@@ -221,7 +212,7 @@ public final class
DataMatchDataConsistencyCalculateAlgorithm extends AbstractSt
/**
* Set result set.
- *
+ *
* @param resultSet result set
*/
public void setResultSet(final ResultSet resultSet) {
@@ -235,87 +226,4 @@ public final class
DataMatchDataConsistencyCalculateAlgorithm extends AbstractSt
CloseUtils.closeQuietly(connection);
}
}
-
- /**
- * Calculated result.
- */
- @RequiredArgsConstructor
- @Getter
- public static final class CalculatedResult implements
DataConsistencyCalculatedResult {
-
- @NonNull
- private final Object maxUniqueKeyValue;
-
- private final int recordsCount;
-
- private final Collection<Collection<Object>> records;
-
- @Override
- public Optional<Object> getMaxUniqueKeyValue() {
- return Optional.of(maxUniqueKeyValue);
- }
-
- @SneakyThrows(SQLException.class)
- @Override
- public boolean equals(final Object o) {
- if (null == o) {
- return false;
- }
- if (this == o) {
- return true;
- }
- if (!(o instanceof CalculatedResult)) {
- log.warn("CalculatedResult type not match, o.className={}",
o.getClass().getName());
- return false;
- }
- final CalculatedResult that = (CalculatedResult) o;
- if (recordsCount != that.recordsCount ||
!Objects.equals(maxUniqueKeyValue, that.maxUniqueKeyValue)) {
- log.warn("recordCount or maxUniqueKeyValue not match,
recordCount1={}, recordCount2={}, maxUniqueKeyValue1={}, maxUniqueKeyValue2={}",
- recordsCount, that.recordsCount, maxUniqueKeyValue,
that.maxUniqueKeyValue);
- return false;
- }
- EqualsBuilder equalsBuilder = new EqualsBuilder();
- Iterator<Collection<Object>> thisIterator = records.iterator();
- Iterator<Collection<Object>> thatIterator =
that.records.iterator();
- while (thisIterator.hasNext() && thatIterator.hasNext()) {
- equalsBuilder.reset();
- Collection<Object> thisNext = thisIterator.next();
- Collection<Object> thatNext = thatIterator.next();
- if (thisNext.size() != thatNext.size()) {
- log.warn("record column size not match, size1={},
size2={}, record1={}, record2={}", thisNext.size(), thatNext.size(), thisNext,
thatNext);
- return false;
- }
- Iterator<Object> thisNextIterator = thisNext.iterator();
- Iterator<Object> thatNextIterator = thatNext.iterator();
- int columnIndex = 0;
- while (thisNextIterator.hasNext() &&
thatNextIterator.hasNext()) {
- ++columnIndex;
- Object thisResult = thisNextIterator.next();
- Object thatResult = thatNextIterator.next();
- boolean matched;
- if (thisResult instanceof SQLXML && thatResult instanceof
SQLXML) {
- matched = ((SQLXML)
thisResult).getString().equals(((SQLXML) thatResult).getString());
- } else if (thisResult instanceof BigDecimal && thatResult
instanceof BigDecimal) {
- matched =
DataConsistencyCheckUtils.isBigDecimalEquals((BigDecimal) thisResult,
(BigDecimal) thatResult);
- } else if (thisResult instanceof Array && thatResult
instanceof Array) {
- matched = Objects.deepEquals(((Array)
thisResult).getArray(), ((Array) thatResult).getArray());
- } else {
- matched = equalsBuilder.append(thisResult,
thatResult).isEquals();
- }
- if (!matched) {
- log.warn("record column value not match,
columnIndex={}, value1={}, value2={}, value1.class={}, value2.class={},
record1={}, record2={}", columnIndex, thisResult, thatResult,
- null != thisResult ?
thisResult.getClass().getName() : "", null != thatResult ?
thatResult.getClass().getName() : "",
- thisNext, thatNext);
- return false;
- }
- }
- }
- return true;
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17,
37).append(getMaxUniqueKeyValue().orElse(null)).append(getRecordsCount()).append(getRecords()).toHashCode();
- }
- }
}
diff --git
a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchCalculatedResultTest.java
b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResultTest.java
similarity index 74%
rename from
kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchCalculatedResultTest.java
rename to
kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResultTest.java
index e7933bce52b..ba6412caf68 100644
---
a/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/algorithm/DataMatchCalculatedResultTest.java
+++
b/kernel/data-pipeline/core/src/test/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataMatchCalculatedResultTest.java
@@ -15,9 +15,8 @@
* limitations under the License.
*/
-package
org.apache.shardingsphere.data.pipeline.core.check.consistency.algorithm;
+package org.apache.shardingsphere.data.pipeline.core.check.consistency;
-import
org.apache.shardingsphere.data.pipeline.core.check.consistency.algorithm.DataMatchDataConsistencyCalculateAlgorithm.CalculatedResult;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
@@ -37,15 +36,15 @@ class DataMatchCalculatedResultTest {
@Test
void assertEmptyRecordsEquals() {
- CalculatedResult actual = new CalculatedResult(0, 0,
Collections.emptyList());
- CalculatedResult expected = new CalculatedResult(0, 0,
Collections.emptyList());
+ DataMatchCalculatedResult actual = new DataMatchCalculatedResult(0, 0,
Collections.emptyList());
+ DataMatchCalculatedResult expected = new DataMatchCalculatedResult(0,
0, Collections.emptyList());
assertThat(actual, is(expected));
}
@Test
void assertFullTypeRecordsEquals() {
- CalculatedResult actual = new CalculatedResult(1000, 2,
Arrays.asList(buildFixedFullTypeRecord(), buildFixedFullTypeRecord()));
- CalculatedResult expected = new CalculatedResult(1000, 2,
Arrays.asList(buildFixedFullTypeRecord(), buildFixedFullTypeRecord()));
+ DataMatchCalculatedResult actual = new DataMatchCalculatedResult(1000,
2, Arrays.asList(buildFixedFullTypeRecord(), buildFixedFullTypeRecord()));
+ DataMatchCalculatedResult expected = new
DataMatchCalculatedResult(1000, 2, Arrays.asList(buildFixedFullTypeRecord(),
buildFixedFullTypeRecord()));
assertThat(actual, is(expected));
}
@@ -56,7 +55,7 @@ class DataMatchCalculatedResultTest {
@Test
void assertFullTypeRecordsEqualsWithDifferentDecimalScale() {
- CalculatedResult expected = new CalculatedResult(1000, 1,
Collections.singletonList(buildFixedFullTypeRecord()));
+ DataMatchCalculatedResult expected = new
DataMatchCalculatedResult(1000, 1,
Collections.singletonList(buildFixedFullTypeRecord()));
List<Object> record = buildFixedFullTypeRecord();
for (int index = 0; index < record.size(); index++) {
if (record.get(index) instanceof BigDecimal) {
@@ -64,30 +63,30 @@ class DataMatchCalculatedResultTest {
record.set(index, decimal.setScale(decimal.scale() + 1,
RoundingMode.CEILING));
}
}
- CalculatedResult actual = new CalculatedResult(1000, 1,
Collections.singletonList(record));
+ DataMatchCalculatedResult actual = new DataMatchCalculatedResult(1000,
1, Collections.singletonList(record));
assertThat(actual, is(expected));
}
@Test
void assertRecordsCountNotEquals() {
- CalculatedResult result1 = new CalculatedResult(1000, 1,
Collections.emptyList());
- CalculatedResult result2 = new CalculatedResult(1000, 0,
Collections.emptyList());
+ DataMatchCalculatedResult result1 = new
DataMatchCalculatedResult(1000, 1, Collections.emptyList());
+ DataMatchCalculatedResult result2 = new
DataMatchCalculatedResult(1000, 0, Collections.emptyList());
assertNotEquals(result1, result2);
}
@Test
void assertMaxUniqueKeyValueNotEquals() {
- CalculatedResult result1 = new CalculatedResult(1000, 1,
Collections.emptyList());
- CalculatedResult result2 = new CalculatedResult(1001, 1,
Collections.emptyList());
+ DataMatchCalculatedResult result1 = new
DataMatchCalculatedResult(1000, 1, Collections.emptyList());
+ DataMatchCalculatedResult result2 = new
DataMatchCalculatedResult(1001, 1, Collections.emptyList());
assertNotEquals(result1, result2);
}
@Test
void assertRandomColumnValueNotEquals() {
List<Object> record = buildFixedFullTypeRecord();
- CalculatedResult result1 = new CalculatedResult(1000, 1,
Collections.singletonList(record));
+ DataMatchCalculatedResult result1 = new
DataMatchCalculatedResult(1000, 1, Collections.singletonList(record));
for (int index = 0; index < record.size(); index++) {
- CalculatedResult result2 = new CalculatedResult(1000, 1,
Collections.singletonList(modifyColumnValueRandomly(buildFixedFullTypeRecord(),
index)));
+ DataMatchCalculatedResult result2 = new
DataMatchCalculatedResult(1000, 1,
Collections.singletonList(modifyColumnValueRandomly(buildFixedFullTypeRecord(),
index)));
assertNotEquals(result1, result2);
}
}
