Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2422#discussion_r199559152
--- Diff:
integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
---
@@ -299,17 +303,112 @@ class AlterTableColumnSchemaGenerator(
val columnValidator = CarbonSparkFactory.getCarbonColumnValidator
columnValidator.validateColumns(allColumns)
+
+ def getLocalDictColumnList(tableProperties:
scala.collection.mutable.Map[String, String],
+ columns: scala.collection.mutable.ListBuffer[ColumnSchema]):
(scala.collection.mutable
+ .ListBuffer[ColumnSchema],
scala.collection.mutable.ListBuffer[ColumnSchema]) = {
+ val includeColumns = new
scala.collection.mutable.ListBuffer[ColumnSchema]
+ val excludeColumns = new
scala.collection.mutable.ListBuffer[ColumnSchema]
+ val localDictIncludeColumns = if (tableProperties
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE).isDefined) {
+ tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)
+ } else {
+ null
+ }
+ val localDictExcludeColumns = if (tableProperties
+ .get(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE).isDefined) {
+ tableProperties(CarbonCommonConstants.LOCAL_DICTIONARY_EXCLUDE)
+ } else {
+ null
+ }
+ if (null != localDictIncludeColumns) {
+ if (null == localDictExcludeColumns) {
+ columns.foreach { column =>
+ if (localDictIncludeColumns.contains(column.getColumnName)) {
+ includeColumns.append(column)
+ } else {
+ if (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.toString.equals("ARRAY") ||
+ column.getDataType.toString.equals("STRUCT")) {
+ excludeColumns.append(column)
+ }
+ }
+ }
+ } else {
+ columns.foreach { column =>
+ if (localDictIncludeColumns.contains(column.getColumnName) &&
+ !localDictExcludeColumns.contains(column.getColumnName)) {
+ includeColumns.append(column)
+ } else if
(localDictExcludeColumns.contains(column.getColumnName)) {
+ excludeColumns.append(column)
+ }
+ }
+ }
+ } else {
+ if (null == localDictExcludeColumns) {
+ columns.foreach { column =>
+ if (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.toString.equals("ARRAY") ||
+ column.getDataType.toString.equals("STRUCT")) {
+ includeColumns.append(column)
+ }
+ }
+ } else {
+ columns.foreach { column =>
+ if (!localDictExcludeColumns.contains(column.getColumnName) &&
+ (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.toString.equals("ARRAY") ||
+ column.getDataType.toString.equals("STRUCT"))) {
+ includeColumns.append(column)
+ } else if
(localDictExcludeColumns.contains(column.getColumnName)) {
+ excludeColumns.append(column)
+ }
+ }
+ }
+ }
+
+ (includeColumns, excludeColumns)
+ }
+
+ val columnsWithoutNewCols = new
scala.collection.mutable.ListBuffer[ColumnSchema]
+ allColumns.foreach { column =>
+ if (!newCols.exists(x =>
x.getColumnName.equalsIgnoreCase(column.getColumnName))) {
+ columnsWithoutNewCols += column
+ }
+ }
+
+
if (alterTableModel.tableProperties != null) {
CarbonUtil
.setLocalDictColumnsToWrapperSchema(newCols.asJava,
alterTableModel.tableProperties.asJava,
tableSchema.getTableProperties.get(CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE))
}
+ val newMainList =
getLocalDictColumnList(tableSchema.getTableProperties.asScala,
+ columnsWithoutNewCols)
+ val ss: scala.collection.mutable.Map[String, String] = mutable
+ .Map(alterTableModel.tableProperties.toSeq: _*)
+ val newAlterList = getLocalDictColumnList(ss,
+ newCols.to[mutable.ListBuffer])
+
+ newMainList._1.appendAll(newAlterList._1)
+ newMainList._2.appendAll(newAlterList._2)
+
+ val localDictioanryIncludeColumn = new StringBuilder
+ val localDictioanryExcludeColumn = new StringBuilder
+ newMainList._1.foreach { column =>
+ localDictioanryIncludeColumn.append(column.getColumnName).append(",")
+ }
+ newMainList._2.foreach { column =>
+ localDictioanryExcludeColumn.append(column.getColumnName).append(",")
+ }
+
// populate table properties map
val tablePropertiesMap = tableSchema.getTableProperties
alterTableModel.tableProperties.foreach {
- x => val value = tablePropertiesMap.get(x._1)
+ x =>
--- End diff --
.use a tuple of (key, value) instead of x like .foreach { (key, value) => }
---
