[shardingsphere] branch master updated: Fix data consistency check failed when table with only unique keys (#24040)

[zhonghongsheng]

This is an automated email from the ASF dual-hosted git repository.

zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git


The following commit(s) were added to refs/heads/master by this push:
     new 216df382945 Fix data consistency check failed when table with only 
unique keys (#24040)
216df382945 is described below

commit 216df3829455db1c9b81adefc12e631eba9b8e94
Author: Xinze Guo <101622833+aze...@users.noreply.github.com>
AuthorDate: Tue Feb 7 14:56:28 2023 +0800

    Fix data consistency check failed when table with only unique keys (#24040)
---
 .../pipeline/core/metadata/loader/PipelineTableMetaDataUtil.java     | 5 ++---
 .../e2e/data/pipeline/cases/migration/AbstractMigrationE2EIT.java    | 5 ++++-
 .../cases/migration/general/PostgreSQLMigrationGeneralE2EIT.java     | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git 
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/metadata/loader/PipelineTableMetaDataUtil.java
 
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/metadata/loader/PipelineTableMetaDataUtil.java
index 69a2a108d8e..ff652e93280 100644
--- 
a/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/metadata/loader/PipelineTableMetaDataUtil.java
+++ 
b/kernel/data-pipeline/core/src/main/java/org/apache/shardingsphere/data/pipeline/core/metadata/loader/PipelineTableMetaDataUtil.java
@@ -55,11 +55,10 @@ public final class PipelineTableMetaDataUtil {
         if (1 == primaryKeys.size()) {
             return 
tableMetaData.getColumnMetaData(tableMetaData.getPrimaryKeyColumns().get(0));
         }
-        if (primaryKeys.isEmpty()) {
+        Collection<PipelineIndexMetaData> uniqueIndexes = 
tableMetaData.getUniqueIndexes();
+        if (uniqueIndexes.isEmpty() && primaryKeys.isEmpty()) {
             return null;
         }
-        Collection<PipelineIndexMetaData> uniqueIndexes = 
tableMetaData.getUniqueIndexes();
-        ShardingSpherePreconditions.checkState(!uniqueIndexes.isEmpty(), () -> 
new SplitPipelineJobByRangeException(tableName, "no primary key or unique 
index"));
         if (1 == uniqueIndexes.size() && 1 == 
uniqueIndexes.iterator().next().getColumns().size()) {
             PipelineColumnMetaData column = 
uniqueIndexes.iterator().next().getColumns().get(0);
             if (!column.isNullable()) {
diff --git 
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/AbstractMigrationE2EIT.java
 
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/AbstractMigrationE2EIT.java
index d668060cba5..9cdc6d41b69 100644
--- 
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/AbstractMigrationE2EIT.java
+++ 
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/AbstractMigrationE2EIT.java
@@ -39,6 +39,7 @@ import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 @Getter
@@ -150,6 +151,7 @@ public abstract class AbstractMigrationE2EIT extends 
PipelineBaseE2EIT {
             resultList = queryForListWithLog(String.format("SHOW MIGRATION 
CHECK STATUS '%s'", jobId));
             if (resultList.isEmpty()) {
                 ThreadUtil.sleep(3, TimeUnit.SECONDS);
+                continue;
             }
             List<String> checkEndTimeList = resultList.stream().map(map -> 
map.get("check_end_time").toString()).filter(each -> 
!Strings.isNullOrEmpty(each)).collect(Collectors.toList());
             if (checkEndTimeList.size() == resultList.size()) {
@@ -159,8 +161,9 @@ public abstract class AbstractMigrationE2EIT extends 
PipelineBaseE2EIT {
             }
         }
         log.info("check job results: {}", resultList);
+        assertFalse(resultList.isEmpty());
         for (Map<String, Object> each : resultList) {
-            assertTrue("check result is false", 
Boolean.parseBoolean(each.get("result").toString()));
+            assertTrue(String.format("%s check result is false", 
each.get("tables")), Boolean.parseBoolean(each.get("result").toString()));
             assertThat("finished_percentage is not 100", 
each.get("finished_percentage").toString(), is("100"));
         }
     }
diff --git 
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/PostgreSQLMigrationGeneralE2EIT.java
 
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/PostgreSQLMigrationGeneralE2EIT.java
index 4a1c65f0e48..d1b44b688a1 100644
--- 
a/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/PostgreSQLMigrationGeneralE2EIT.java
+++ 
b/test/e2e/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/data/pipeline/cases/migration/general/PostgreSQLMigrationGeneralE2EIT.java
@@ -101,7 +101,7 @@ public final class PostgreSQLMigrationGeneralE2EIT extends 
AbstractMigrationE2EI
         }
         List<String> lastJobIds = listJobId();
         assertThat(lastJobIds.size(), is(0));
-        
assertGreaterThanOrderTableInitRows(PipelineBaseE2EIT.TABLE_INIT_ROW_COUNT, 
PipelineBaseE2EIT.SCHEMA_NAME);
+        
assertGreaterThanOrderTableInitRows(PipelineBaseE2EIT.TABLE_INIT_ROW_COUNT + 1, 
PipelineBaseE2EIT.SCHEMA_NAME);
         log.info("{} E2E IT finished, database type={}, docker image={}", 
this.getClass().getName(), testParam.getDatabaseType(), 
testParam.getStorageContainerImage());
     }