This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 7958c51 For #14604：Fix "Data truncated for column" error occurs
during the migration of the year type field (#14688)
7958c51 is described below
commit 7958c518b469a1abe1b9c39dc18fd26b34ff61ca
Author: ReyYang <[email protected]>
AuthorDate: Tue Jan 11 20:30:05 2022 +0800
For #14604：Fix "Data truncated for column" error occurs during the
migration of the year type field (#14688)
* For #14604：Fix "Data truncated for column" error occurs during the
migration of the year type field
* For #14604：Fix review
---
.../query/impl/driver/jdbc/type/memory/JDBCRowsLoader.java | 10 ++++++++++
.../core/check/consistency/DataConsistencyCheckerImpl.java | 12 ++++++++++++
.../data/pipeline/mysql/ingest/MySQLInventoryDumper.java | 11 ++++++++++-
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git
a/shardingsphere-infra/shardingsphere-infra-executor/src/main/java/org/apache/shardingsphere/infra/executor/sql/execute/result/query/impl/driver/jdbc/type/memory/JDBCRowsLoader.java
b/shardingsphere-infra/shardingsphere-infra-executor/src/main/java/org/apache/shardingsphere/infra/executor/sql/execute/result/query/impl/driver/jdbc/type/memory/JDBCRowsLoader.java
index fdc111c..2ecc55b 100644
---
a/shardingsphere-infra/shardingsphere-infra-executor/src/main/java/org/apache/shardingsphere/infra/executor/sql/execute/result/query/impl/driver/jdbc/type/memory/JDBCRowsLoader.java
+++
b/shardingsphere-infra/shardingsphere-infra-executor/src/main/java/org/apache/shardingsphere/infra/executor/sql/execute/result/query/impl/driver/jdbc/type/memory/JDBCRowsLoader.java
@@ -37,6 +37,8 @@ import java.util.List;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public final class JDBCRowsLoader {
+ private static final String YEAR_DATA_TYPE = "YEAR";
+
/**
* Load rows.
*
@@ -89,6 +91,10 @@ public final class JDBCRowsLoader {
case Types.LONGVARCHAR:
return resultSet.getString(columnIndex);
case Types.DATE:
+ if
(isYearDataType(resultSet.getMetaData().getColumnTypeName(columnIndex))) {
+ Object result = resultSet.getObject(columnIndex);
+ return resultSet.wasNull() ? null : result;
+ }
return resultSet.getDate(columnIndex);
case Types.TIME:
return resultSet.getTime(columnIndex);
@@ -108,4 +114,8 @@ public final class JDBCRowsLoader {
return resultSet.getObject(columnIndex);
}
}
+
+ private static boolean isYearDataType(final String columnDataTypeName) {
+ return YEAR_DATA_TYPE.equalsIgnoreCase(columnDataTypeName);
+ }
}
diff --git
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
index 565d605..d4c3885 100644
---
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
+++
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-core/src/main/java/org/apache/shardingsphere/data/pipeline/core/check/consistency/DataConsistencyCheckerImpl.java
@@ -35,6 +35,7 @@ import
org.apache.shardingsphere.data.pipeline.spi.check.consistency.DataConsist
import
org.apache.shardingsphere.data.pipeline.spi.check.consistency.SingleTableDataCalculator;
import
org.apache.shardingsphere.data.pipeline.spi.ratelimit.JobRateLimitAlgorithm;
import org.apache.shardingsphere.infra.database.type.DatabaseType;
+import org.apache.shardingsphere.infra.database.type.dialect.MySQLDatabaseType;
import
org.apache.shardingsphere.infra.executor.kernel.thread.ExecutorThreadFactoryBuilder;
import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
import org.apache.shardingsphere.infra.metadata.schema.model.TableMetaData;
@@ -52,6 +53,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
+import java.util.Properties;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -136,6 +138,7 @@ public final class DataConsistencyCheckerImpl implements
DataConsistencyChecker
PipelineDataSourceConfiguration targetDataSourceConfig =
PipelineDataSourceConfigurationFactory.newInstance(
jobContext.getJobConfig().getPipelineConfig().getTarget().getType(),
jobContext.getJobConfig().getPipelineConfig().getTarget().getParameter());
checkDatabaseTypeSupportedOrNot(supportedDatabaseTypes,
targetDataSourceConfig.getDatabaseType().getName());
+ addDataSourceConfigToMySQL(sourceDataSourceConfig,
targetDataSourceConfig);
Collection<String> logicTableNames =
jobContext.getTaskConfigs().stream().flatMap(each ->
each.getDumperConfig().getTableNameMap().values().stream()).distinct().collect(Collectors.toList());
String sourceDatabaseType =
sourceDataSourceConfig.getDatabaseType().getName();
String targetDatabaseType =
targetDataSourceConfig.getDatabaseType().getName();
@@ -201,4 +204,13 @@ public final class DataConsistencyCheckerImpl implements
DataConsistencyChecker
ShardingSphereMetaData metaData =
contextManager.getMetaDataContexts().getMetaData(schemaName);
return metaData.getSchema().getTables();
}
+
+ private void addDataSourceConfigToMySQL(final
PipelineDataSourceConfiguration sourceDataSourceConfig, final
PipelineDataSourceConfiguration targetDataSourceConfig) {
+ if
(sourceDataSourceConfig.getDatabaseType().getName().equalsIgnoreCase(new
MySQLDatabaseType().getName())) {
+ Properties queryProps = new Properties();
+ queryProps.setProperty("yearIsDateType", Boolean.FALSE.toString());
+ sourceDataSourceConfig.appendJDBCQueryProperties(queryProps);
+ targetDataSourceConfig.appendJDBCQueryProperties(queryProps);
+ }
+ }
}
diff --git
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-dialect/shardingsphere-data-pipeline-mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLInventoryDumper.java
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-dialect/shardingsphere-data-pipeline-mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLInventoryDumper.java
index 9559034..8f62b57 100644
---
a/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-dialect/shardingsphere-data-pipeline-mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLInventoryDumper.java
+++
b/shardingsphere-kernel/shardingsphere-data-pipeline/shardingsphere-data-pipeline-dialect/shardingsphere-data-pipeline-mysql/src/main/java/org/apache/shardingsphere/data/pipeline/mysql/ingest/MySQLInventoryDumper.java
@@ -33,6 +33,8 @@ import java.util.Properties;
*/
public final class MySQLInventoryDumper extends AbstractInventoryDumper {
+ private static final String YEAR_DATA_TYPE = "YEAR";
+
public MySQLInventoryDumper(final InventoryDumperConfiguration
inventoryDumperConfig, final PipelineDataSourceManager dataSourceManager) {
super(inventoryDumperConfig, dataSourceManager);
Properties queryProps = new Properties();
@@ -42,7 +44,10 @@ public final class MySQLInventoryDumper extends
AbstractInventoryDumper {
@Override
public Object readValue(final ResultSet resultSet, final int index) throws
SQLException {
- if (isDateTimeValue(resultSet.getMetaData().getColumnType(index))) {
+ if (isYearDataType(resultSet.getMetaData().getColumnTypeName(index))) {
+ Object result = resultSet.getObject(index);
+ return resultSet.wasNull() ? null : result;
+ } else if
(isDateTimeValue(resultSet.getMetaData().getColumnType(index))) {
return resultSet.getString(index);
} else {
return resultSet.getObject(index);
@@ -53,6 +58,10 @@ public final class MySQLInventoryDumper extends
AbstractInventoryDumper {
return Types.TIME == columnType || Types.DATE == columnType ||
Types.TIMESTAMP == columnType;
}
+ private boolean isYearDataType(final String columnDataTypeName) {
+ return YEAR_DATA_TYPE.equalsIgnoreCase(columnDataTypeName);
+ }
+
@Override
protected PreparedStatement createPreparedStatement(final Connection
connection, final String sql) throws SQLException {
PreparedStatement result = connection.prepareStatement(sql,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
