This is an automated email from the ASF dual-hosted git repository. ajantha pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 7caa530 [CARBONDATA-3928] Removed records from exception message. 7caa530 is described below commit 7caa530edde9709f3e7fae0d2f2e2956cf7e77cb Author: Nihal ojha <nihalnit...@gmail.com> AuthorDate: Fri Sep 11 13:01:37 2020 +0530 [CARBONDATA-3928] Removed records from exception message. Why is this PR needed? Currently, when the string length exceeds 32000 and bad record action as default then records are thrown in exception message which violates security concern. What changes were proposed in this PR? Removed the records from exception message and only printed in logger file and redirected csv file. Does this PR introduce any user interface change? No Is any new testcase added? Yes This closes #3921 --- .../carbondata/core/constants/CarbonCommonConstants.java | 2 +- .../org/apache/carbondata/spark/util/CarbonScalaUtil.scala | 2 +- .../spark/testsuite/dataload/TestLoadDataGeneral.scala | 4 ++-- .../carbondata/processing/datatypes/PrimitiveDataType.java | 6 +++--- .../processing/loading/converter/impl/RowConverterImpl.java | 10 ++-------- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index d34f357..e497c73 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -2535,7 +2535,7 @@ public final class CarbonCommonConstants { public static final String CARBON_INDEXSERVER_TEMPFOLDER_DELETETIME_DEFAULT = "10800000"; public static final String STRING_LENGTH_EXCEEDED_MESSAGE = - "Record %s of column %s exceeded " + MAX_CHARS_PER_COLUMN_DEFAULT + + "Record of column %s exceeded " + MAX_CHARS_PER_COLUMN_DEFAULT + " characters. Please consider long string data type."; /** diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala index 2806376..3e1374e 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala @@ -76,7 +76,7 @@ object CarbonScalaUtil { } catch { case e: Exception => if (e.getMessage.startsWith(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE)) { - val msg = CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE.format(row, + val msg = CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE.format( carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getCreateOrderColumn .get(idx).getColName) LOGGER.error(msg, e) diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala index aaaa88d..a5a2c41 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala @@ -236,7 +236,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach { exception = intercept[Exception] { sql(s"insert into longerthan32kchar values('33000', '$longChar', 4)") } - assert(exception.getMessage.contains(s"Record [33000, $longChar, 4] of column dim2 exceeded " + + assert(exception.getMessage.contains(s"Record of column dim2 exceeded " + s"${CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT} characters. Please consider long string data type.")) // Update strings of length greater than 32000 sql(s"insert into longerthan32kchar values('ok', 'hi', 1)") @@ -244,7 +244,7 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterEach { sql(s"update longerthan32kchar set(longerthan32kchar.dim2)=('$longChar') " + "where longerthan32kchar.mes1=1").show() } - assert(exception.getMessage.contains(s"Record [ok, $longChar, 1] of column dim2 exceeded " + + assert(exception.getMessage.contains(s"Record of column dim2 exceeded " + s"${CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT} characters. Please consider long string data type.")) CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_ENABLE_BAD_RECORD_HANDLING_FOR_INSERT, "false") diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java index d9f3e8c..1a09a5d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java @@ -302,7 +302,7 @@ public class PrimitiveDataType implements GenericDataType<Object> { if (this.carbonDimension.getDataType() == DataTypes.STRING && value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) { logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE, - input.toString(), this.carbonDimension.getColName())); + this.carbonDimension.getColName())); updateNullValue(dataOutputStream, logHolder); return; } @@ -339,7 +339,7 @@ public class PrimitiveDataType implements GenericDataType<Object> { if (this.carbonDimension.getDataType() == DataTypes.STRING && input instanceof String && ((String)input).length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) { logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE, - input.toString(), this.carbonDimension.getColName())); + this.carbonDimension.getColName())); updateNullValue(dataOutputStream, logHolder); return; } @@ -348,7 +348,7 @@ public class PrimitiveDataType implements GenericDataType<Object> { if (this.carbonDimension.getDataType() == DataTypes.STRING && value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) { logHolder.setReason(String.format(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE, - input.toString(), this.carbonDimension.getColName())); + this.carbonDimension.getColName())); updateNullValue(dataOutputStream, logHolder); return; } diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java index 5d2226a..28bfa91 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/RowConverterImpl.java @@ -18,7 +18,6 @@ package org.apache.carbondata.processing.loading.converter.impl; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -119,10 +118,6 @@ public class RowConverterImpl implements RowConverter { .getTableProperties(); String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX); boolean isSpatialColumn = false; - Object[] rawData = row.getRawData(); - if (rawData == null) { - rawData = row.getData() == null ? null : row.getData().clone(); - } for (int i = 0; i < fieldConverters.length; i++) { if (spatialProperty != null) { isSpatialColumn = fieldConverters[i].getDataField().getColumn().getColName() @@ -137,10 +132,9 @@ public class RowConverterImpl implements RowConverter { if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) { String reason = logHolder.getReason(); if (reason.equalsIgnoreCase(CarbonCommonConstants.STRING_LENGTH_EXCEEDED_MESSAGE)) { - reason = String.format(reason, Arrays.toString(rawData), - this.fields[i].getColumn().getColName()); + reason = String.format(reason, this.fields[i].getColumn().getColName()); } - badRecordLogger.addBadRecordsToBuilder(rawData, reason); + badRecordLogger.addBadRecordsToBuilder(row.getRawData(), reason); if (badRecordLogger.isDataLoadFail()) { String error = "Data load failed due to bad record: " + reason; if (!badRecordLogger.isBadRecordLoggerEnable()) {