This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ed6e001d926 HIVE-29252: Iceberg: [V3] Add support for Column Defaults
with Alter commands (#6120)
ed6e001d926 is described below
commit ed6e001d9268ccba8ef2c7fee15ca336b5b5a78e
Author: Ayush Saxena <[email protected]>
AuthorDate: Sat Oct 18 19:34:42 2025 +0530
HIVE-29252: Iceberg: [V3] Add support for Column Defaults with Alter
commands (#6120)
---
.../apache/iceberg/hive/HiveSchemaConverter.java | 35 +----
.../org/apache/iceberg/hive/HiveSchemaUtil.java | 114 ++++++++++++++-
.../apache/iceberg/hive/TestHiveSchemaUtil.java | 2 +-
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 91 ++++++++++--
.../iceberg_replace_column_with_default_change.q | 12 ++
.../positive/iceberg_alter_default_column.q | 49 +++++++
...ceberg_replace_column_with_default_change.q.out | 30 ++++
.../positive/iceberg_alter_default_column.q.out | 161 +++++++++++++++++++++
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 13 +-
.../org/apache/hadoop/hive/ql/ddl/DDLUtils.java | 13 ++
.../column/add/AlterTableAddColumnsAnalyzer.java | 10 +-
.../change/AlterTableChangeColumnAnalyzer.java | 21 ++-
.../replace/AlterTableReplaceColumnsAnalyzer.java | 9 +-
.../ql/ddl/table/constraint/ConstraintsUtils.java | 5 +-
.../ql/ddl/table/create/CreateTableAnalyzer.java | 2 +-
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 8 +-
.../hadoop/hive/ql/session/SessionStateUtil.java | 1 +
17 files changed, 506 insertions(+), 70 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index d7891fe7601..330f67e32bb 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -22,7 +22,6 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
@@ -31,11 +30,8 @@
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.Expressions;
-import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.slf4j.Logger;
@@ -63,9 +59,9 @@ static Schema convert(List<String> names, List<TypeInfo>
typeInfos, List<String>
return new Schema(converter.convertInternal(names, typeInfos,
defaultValues, comments));
}
- static Type convert(TypeInfo typeInfo, boolean autoConvert) {
+ public static Type convert(TypeInfo typeInfo, boolean autoConvert, String
defaultValue) {
HiveSchemaConverter converter = new HiveSchemaConverter(autoConvert);
- return converter.convertType(typeInfo, null);
+ return converter.convertType(typeInfo, defaultValue);
}
List<Types.NestedField> convertInternal(List<String> names, List<TypeInfo>
typeInfos,
@@ -86,7 +82,7 @@ List<Types.NestedField> convertInternal(List<String> names,
List<TypeInfo> typeI
if (defaultValues.containsKey(columnName)) {
if (type.isPrimitiveType()) {
- Object icebergDefaultValue =
getDefaultValue(defaultValues.get(columnName), type);
+ Object icebergDefaultValue =
HiveSchemaUtil.getDefaultValue(defaultValues.get(columnName), type);
fieldBuilder.withWriteDefault(Expressions.lit(icebergDefaultValue));
} else if (!type.isStructType()) {
throw new UnsupportedOperationException(
@@ -99,13 +95,6 @@ List<Types.NestedField> convertInternal(List<String> names,
List<TypeInfo> typeI
return result;
}
- private static Object getDefaultValue(String defaultValue, Type type) {
- return switch (type.typeId()) {
- case DATE, TIME, TIMESTAMP, TIMESTAMP_NANO ->
Literal.of(stripQuotes(defaultValue)).to(type).value();
- default -> Conversions.fromPartitionString(type,
stripQuotes(defaultValue));
- };
- }
-
Type convertType(TypeInfo typeInfo, String defaultValue) {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
@@ -162,7 +151,7 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
List<Types.NestedField> fields =
convertInternal(structTypeInfo.getAllStructFieldNames(),
structTypeInfo.getAllStructFieldTypeInfos(),
- getDefaultValuesMap(defaultValue), Collections.emptyList());
+ HiveSchemaUtil.getDefaultValuesMap(defaultValue),
Collections.emptyList());
return Types.StructType.of(fields);
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
@@ -182,20 +171,4 @@ Type convertType(TypeInfo typeInfo, String defaultValue) {
throw new IllegalArgumentException("Unknown type " +
typeInfo.getCategory());
}
}
-
- private static Map<String, String> getDefaultValuesMap(String defaultValue) {
- if (StringUtils.isEmpty(defaultValue)) {
- return Collections.emptyMap();
- }
- // For Struct, the default value is expected to be in key:value format
- return
Splitter.on(',').trimResults().withKeyValueSeparator(':').split(stripQuotes(defaultValue));
- }
-
- public static String stripQuotes(String val) {
- if (val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'' ||
- val.charAt(0) == '"' && val.charAt(val.length() - 1) == '"') {
- return val.substring(1, val.length() - 1);
- }
- return val;
- }
}
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
index ce563b1e55d..362ea6a1006 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
@@ -27,6 +27,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -34,8 +35,11 @@
import org.apache.iceberg.Schema;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.relocated.com.google.common.base.Splitter;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.DateTimeUtil;
@@ -140,23 +144,29 @@ public static TypeInfo convert(Type type) {
/**
* Converts a Hive typeInfo object to an Iceberg type.
- * @param typeInfo The Hive type
+ *
+ * @param typeInfo The Hive type
+ * @param defaultValue the default value for the column, if any
* @return The Iceberg type
*/
- public static Type convert(TypeInfo typeInfo) {
- return HiveSchemaConverter.convert(typeInfo, false);
+ public static Type convert(TypeInfo typeInfo, String defaultValue) {
+ return HiveSchemaConverter.convert(typeInfo, false, defaultValue);
}
/**
* Returns a SchemaDifference containing those fields which are present in
only one of the collections, as well as
* those fields which are present in both (in terms of the name) but their
type or comment has changed.
- * @param minuendCollection Collection of fields to subtract from
+ *
+ * @param minuendCollection Collection of fields to subtract from
* @param subtrahendCollection Collection of fields to subtract
- * @param bothDirections Whether or not to compute the missing fields from
the minuendCollection as well
+ * @param schema the iceberg table schema, if available. Used
to compare default values
+ * @param defaultValues the column default values
+ * @param bothDirections Whether or not to compute the missing fields
from the minuendCollection as well
* @return the difference between the two schemas
*/
public static SchemaDifference getSchemaDiff(Collection<FieldSchema>
minuendCollection,
- Collection<FieldSchema>
subtrahendCollection, boolean bothDirections) {
+ Collection<FieldSchema> subtrahendCollection, Schema schema, Map<String,
String> defaultValues,
+ boolean bothDirections) {
SchemaDifference difference = new SchemaDifference();
for (FieldSchema first : minuendCollection) {
@@ -178,13 +188,61 @@ public static SchemaDifference
getSchemaDiff(Collection<FieldSchema> minuendColl
}
if (bothDirections) {
- SchemaDifference otherWay = getSchemaDiff(subtrahendCollection,
minuendCollection, false);
+ SchemaDifference otherWay = getSchemaDiff(subtrahendCollection,
minuendCollection, null, defaultValues, false);
otherWay.getMissingFromSecond().forEach(difference::addMissingFromFirst);
}
+ if (schema != null) {
+ for (Types.NestedField field : schema.columns()) {
+ if (!isRemovedField(field, difference.getMissingFromFirst())) {
+ getDefaultValDiff(field, defaultValues, difference);
+ }
+ }
+ }
+
return difference;
}
+ private static boolean isRemovedField(Types.NestedField field,
List<FieldSchema> missingFields) {
+ for (FieldSchema fieldSchema : missingFields) {
+ if (fieldSchema.getName().equalsIgnoreCase(field.name())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Computes whether the default value has changed for the given field.
+ * @param field the field to check for default value change
+ * @param defaultValues the default values for the table schema, if
available. Used to compare default values
+ * @param difference the SchemaDifference object to update with the
default value change if any
+ */
+ private static void getDefaultValDiff(Types.NestedField field, Map<String,
String> defaultValues,
+ SchemaDifference difference) {
+
+ String defaultStr = defaultValues.get(field.name());
+
+ // Skip if no default at all
+ if (defaultStr == null && field.writeDefault() == null) {
+ return;
+ }
+
+ if (field.type().isPrimitiveType()) {
+ Object expectedDefault = HiveSchemaUtil.getDefaultValue(defaultStr,
field.type());
+ if (!Objects.equals(expectedDefault, field.writeDefault())) {
+ difference.addDefaultChanged(field, expectedDefault);
+ }
+ } else if (field.type().isStructType()) {
+ Map<String, String> structDefaults = getDefaultValuesMap(defaultStr);
+
+ for (Types.NestedField nested : field.type().asStructType().fields()) {
+ getDefaultValDiff(nested, structDefaults, difference);
+ }
+ }
+ }
+
+
/**
* Compares two lists of columns to each other to find the (singular) column
that was moved. This works ideally for
* identifying the column that was moved by an ALTER TABLE ... CHANGE COLUMN
command.
@@ -248,6 +306,7 @@ public static class SchemaDifference {
private final List<FieldSchema> missingFromSecond = Lists.newArrayList();
private final List<FieldSchema> typeChanged = Lists.newArrayList();
private final List<FieldSchema> commentChanged = Lists.newArrayList();
+ private final Map<Types.NestedField, Object> defaultChanged =
Maps.newHashMap();
public List<FieldSchema> getMissingFromFirst() {
return missingFromFirst;
@@ -265,9 +324,13 @@ public List<FieldSchema> getCommentChanged() {
return commentChanged;
}
+ public Map<Types.NestedField, Object> getDefaultChanged() {
+ return defaultChanged;
+ }
+
public boolean isEmpty() {
return missingFromFirst.isEmpty() && missingFromSecond.isEmpty() &&
typeChanged.isEmpty() &&
- commentChanged.isEmpty();
+ commentChanged.isEmpty() && defaultChanged.isEmpty();
}
void addMissingFromFirst(FieldSchema field) {
@@ -285,6 +348,10 @@ void addTypeChanged(FieldSchema field) {
void addCommentChanged(FieldSchema field) {
commentChanged.add(field);
}
+
+ void addDefaultChanged(Types.NestedField field, Object defaultValue) {
+ defaultChanged.put(field, defaultValue);
+ }
}
@@ -408,4 +475,35 @@ public static Object convertToWriteType(Object value, Type
type) {
return value; // fallback
}
+
+ public static Map<String, String> getDefaultValuesMap(String defaultValue) {
+ if (StringUtils.isEmpty(defaultValue)) {
+ return Collections.emptyMap();
+ }
+ // For Struct, the default value is expected to be in key:value format
+ return
Splitter.on(',').trimResults().withKeyValueSeparator(':').split(stripQuotes(defaultValue));
+ }
+
+ public static String stripQuotes(String val) {
+ if (val.charAt(0) == '\'' && val.charAt(val.length() - 1) == '\'' ||
+ val.charAt(0) == '"' && val.charAt(val.length() - 1) == '"') {
+ return val.substring(1, val.length() - 1);
+ }
+ return val;
+ }
+
+ public static Object getDefaultValue(String defaultValue, Type type) {
+ if (defaultValue == null) {
+ return null;
+ }
+ return switch (type.typeId()) {
+ case DATE, TIME, TIMESTAMP, TIMESTAMP_NANO ->
+ Literal.of(stripQuotes(defaultValue)).to(type).value();
+ default -> Conversions.fromPartitionString(type,
stripQuotes(defaultValue));
+ };
+ }
+
+ public static Type getStructType(TypeInfo typeInfo, String defaultValue) {
+ return HiveSchemaConverter.convert(typeInfo, false, defaultValue);
+ }
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
index 1b2bae823ce..6daf3aeca5d 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
@@ -220,7 +220,7 @@ private void checkConvert(TypeInfo typeInfo, Type type) {
// Convert to TypeInfo
assertThat(HiveSchemaUtil.convert(type)).isEqualTo(typeInfo);
// Convert to Type
- assertEquals(type, HiveSchemaUtil.convert(typeInfo));
+ assertEquals(type, HiveSchemaUtil.convert(typeInfo, null));
}
/**
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 6c1a696f5e2..77d9915adde 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -31,6 +31,7 @@
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.hadoop.conf.Configuration;
@@ -45,6 +46,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.RequestPartsSpec;
+import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
@@ -102,6 +104,7 @@
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.expressions.UnboundTerm;
@@ -163,6 +166,7 @@ public class HiveIcebergMetaHook extends
BaseHiveIcebergMetaHook {
private Transaction transaction;
private AlterTableType currentAlterTableOp;
private HiveLock commitLock;
+ private List<SQLDefaultConstraint> sqlDefaultConstraints;
public HiveIcebergMetaHook(Configuration conf) {
super(conf);
@@ -502,6 +506,7 @@ public void
commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable
if (transaction != null) {
transaction.commitTransaction();
}
+ setSqlDefaultConstraints();
break;
case ADDPROPS:
case DROPPROPS:
@@ -518,6 +523,16 @@ public void
commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable
}
}
+ private void setSqlDefaultConstraints() {
+ try {
+ if (sqlDefaultConstraints != null && !sqlDefaultConstraints.isEmpty()) {
+
SessionState.get().getHiveDb().addDefaultConstraint(sqlDefaultConstraints);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
@Override
public void rollbackAlterTable(org.apache.hadoop.hive.metastore.api.Table
hmsTable, EnvironmentContext context) {
if (commitLock == null) {
@@ -654,15 +669,24 @@ private void setFileFormat(String format) {
private void handleAddColumns(org.apache.hadoop.hive.metastore.api.Table
hmsTable) {
Collection<FieldSchema> addedCols =
- HiveSchemaUtil.getSchemaDiff(hmsTable.getSd().getCols(),
HiveSchemaUtil.convert(icebergTable.schema()), false)
+ HiveSchemaUtil.getSchemaDiff(hmsTable.getSd().getCols(),
HiveSchemaUtil.convert(icebergTable.schema()), null,
+ null, false)
.getMissingFromSecond();
if (!addedCols.isEmpty()) {
transaction = icebergTable.newTransaction();
updateSchema = transaction.updateSchema();
}
+ this.sqlDefaultConstraints =
+ (List<SQLDefaultConstraint>) SessionStateUtil.getResource(conf,
SessionStateUtil.COLUMN_DEFAULTS).orElse(null);
+ Map<String, String> defaultValues =
Stream.ofNullable(sqlDefaultConstraints).flatMap(Collection::stream)
+ .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name,
SQLDefaultConstraint::getDefault_value));
for (FieldSchema addedCol : addedCols) {
- updateSchema.addColumn(addedCol.getName(),
-
HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType())),
addedCol.getComment());
+ String defaultValue = defaultValues.get(addedCol.getName());
+ Type type =
HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(addedCol.getType()),
defaultValue);
+ Literal<Object> defaultVal = Optional.ofNullable(defaultValue).filter(v
-> !type.isStructType())
+ .map(v -> Expressions.lit(HiveSchemaUtil.getDefaultValue(v,
type))).orElse(null);
+
+ updateSchema.addColumn(addedCol.getName(), type, addedCol.getComment(),
defaultVal);
}
updateSchema.commit();
}
@@ -671,7 +695,8 @@ private void
handleDropColumn(org.apache.hadoop.hive.metastore.api.Table hmsTabl
List<FieldSchema> hmsCols = hmsTable.getSd().getCols();
List<FieldSchema> icebergCols =
HiveSchemaUtil.convert(icebergTable.schema());
- List<FieldSchema> removedCols = HiveSchemaUtil.getSchemaDiff(icebergCols,
hmsCols, false).getMissingFromSecond();
+ List<FieldSchema> removedCols =
+ HiveSchemaUtil.getSchemaDiff(icebergCols, hmsCols, null, null,
false).getMissingFromSecond();
if (removedCols.isEmpty()) {
return;
}
@@ -685,7 +710,14 @@ private void
handleDropColumn(org.apache.hadoop.hive.metastore.api.Table hmsTabl
private void handleReplaceColumns(org.apache.hadoop.hive.metastore.api.Table
hmsTable) throws MetaException {
List<FieldSchema> hmsCols = hmsTable.getSd().getCols();
List<FieldSchema> icebergCols =
HiveSchemaUtil.convert(icebergTable.schema());
- HiveSchemaUtil.SchemaDifference schemaDifference =
HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, true);
+
+ List<SQLDefaultConstraint> defaultConstraints =
+ (List<SQLDefaultConstraint>) SessionStateUtil.getResource(conf,
SessionStateUtil.COLUMN_DEFAULTS).orElse(null);
+ Map<String, String> defaultValues =
Optional.ofNullable(defaultConstraints).orElse(Collections.emptyList()).stream()
+ .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name,
SQLDefaultConstraint::getDefault_value));
+
+ HiveSchemaUtil.SchemaDifference schemaDifference =
+ HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols,
icebergTable.schema(), defaultValues, true);
// if there are columns dropped, let's remove them from the iceberg schema
as well so we can compare the order
if (!schemaDifference.getMissingFromFirst().isEmpty()) {
@@ -697,9 +729,11 @@ private void
handleReplaceColumns(org.apache.hadoop.hive.metastore.api.Table hms
// limit the scope of this operation to only dropping columns
if (!schemaDifference.getMissingFromSecond().isEmpty() ||
!schemaDifference.getTypeChanged().isEmpty() ||
- !schemaDifference.getCommentChanged().isEmpty() || outOfOrder != null)
{
+ !schemaDifference.getCommentChanged().isEmpty() || outOfOrder != null
||
+ !schemaDifference.getDefaultChanged().isEmpty()) {
throw new MetaException("Unsupported operation to use REPLACE COLUMNS
for adding a column, changing a " +
- "column type, column comment or reordering columns. Only use REPLACE
COLUMNS for dropping columns. " +
+ "column type, column comment, column default or reordering columns.
" +
+ "Only use REPLACE COLUMNS for dropping columns. " +
"For the other operations, consider using the ADD COLUMNS or CHANGE
COLUMN commands.");
}
@@ -722,7 +756,12 @@ private void
handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa
List<FieldSchema> hmsCols = hmsTable.getSd().getCols();
List<FieldSchema> icebergCols =
HiveSchemaUtil.convert(icebergTable.schema());
// compute schema difference for renames, type/comment changes
- HiveSchemaUtil.SchemaDifference schemaDifference =
HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, true);
+ List<SQLDefaultConstraint> defaultConstraints =
+ (List<SQLDefaultConstraint>) SessionStateUtil.getResource(conf,
SessionStateUtil.COLUMN_DEFAULTS).orElse(null);
+ Map<String, String> defaultValues =
Optional.ofNullable(defaultConstraints).orElse(Collections.emptyList()).stream()
+ .collect(Collectors.toMap(SQLDefaultConstraint::getColumn_name,
SQLDefaultConstraint::getDefault_value));
+ HiveSchemaUtil.SchemaDifference schemaDifference =
+ HiveSchemaUtil.getSchemaDiff(hmsCols, icebergCols, null, null, true);
// check column reorder (which could happen even in the absence of any
rename, type or comment change)
Map<String, String> renameMapping = ImmutableMap.of();
if (!schemaDifference.getMissingFromSecond().isEmpty()) {
@@ -732,12 +771,12 @@ private void
handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa
}
Pair<String, Optional<String>> outOfOrder =
HiveSchemaUtil.getReorderedColumn(hmsCols, icebergCols, renameMapping);
- if (!schemaDifference.isEmpty() || outOfOrder != null) {
+ if (!schemaDifference.isEmpty() || outOfOrder != null ||
!defaultValues.isEmpty()) {
transaction = icebergTable.newTransaction();
updateSchema = transaction.updateSchema();
} else {
// we should get here if the user didn't change anything about the column
- // i.e. no changes to the name, type, comment or order
+ // i.e. no changes to the name, type, comment, default or order
LOG.info("Found no difference between new and old schema for ALTER TABLE
CHANGE COLUMN for" +
" table: {}. There will be no Iceberg commit.",
hmsTable.getTableName());
return;
@@ -776,11 +815,41 @@ private void
handleChangeColumn(org.apache.hadoop.hive.metastore.api.Table hmsTa
updateSchema.moveFirst(outOfOrder.first());
}
}
+
+ // case 5: handle change of default values
+ handleDefaultValues(defaultValues, renameMapping,
icebergTable.schema().columns(), "");
updateSchema.commit();
handlePartitionRename(schemaDifference);
}
+ /**
+ * Updates the default values of the fields.
+ * @param defaultValues the map containing the default values.
+ * @param renameMapping the rename mapping of the columns
+ * @param columns the columns of the table
+ * @param prefix the prefix of the columns, empty unless a filed of struct.
+ */
+ private void handleDefaultValues(Map<String, String> defaultValues,
Map<String, String> renameMapping,
+ List<Types.NestedField> columns, String prefix) {
+ if (!defaultValues.isEmpty()) {
+ for (Map.Entry<String, String> field : defaultValues.entrySet()) {
+ String simpleName =
+ renameMapping.containsKey(field.getKey()) ?
renameMapping.get(field.getKey()) : field.getKey();
+ String qualifiedName = prefix + simpleName;
+ Type fieldType =
+ columns.stream().filter(col ->
col.name().equalsIgnoreCase(simpleName)).findFirst().get().type();
+ if (fieldType.isStructType()) {
+ Map<String, String> structDefaults =
HiveSchemaUtil.getDefaultValuesMap(field.getValue());
+ handleDefaultValues(structDefaults, renameMapping,
fieldType.asStructType().fields(), qualifiedName + ".");
+ } else {
+ updateSchema.updateColumnDefault(qualifiedName,
+ Expressions.lit(HiveSchemaUtil.getDefaultValue(field.getValue(),
fieldType)));
+ }
+ }
+ }
+ }
+
private void handlePartitionRename(HiveSchemaUtil.SchemaDifference
schemaDifference) {
// in case a partition column has been renamed, spec needs to be adjusted
too
if (!schemaDifference.getMissingFromSecond().isEmpty()) {
@@ -795,7 +864,7 @@ private void
handlePartitionRename(HiveSchemaUtil.SchemaDifference schemaDiffere
}
private Type.PrimitiveType getPrimitiveTypeOrThrow(FieldSchema field) throws
MetaException {
- Type newType =
HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType()));
+ Type newType =
HiveSchemaUtil.convert(TypeInfoUtils.getTypeInfoFromTypeString(field.getType()),
null);
if (!(newType instanceof Type.PrimitiveType)) {
throw new MetaException(String.format("Cannot promote type of column:
'%s' to a non-primitive type: %s.",
field.getName(), newType));
diff --git
a/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q
b/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q
new file mode 100644
index 00000000000..3eefaf306b0
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/negative/iceberg_replace_column_with_default_change.q
@@ -0,0 +1,12 @@
+CREATE TABLE ice_t (
+ id INT,
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25
+)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+
+ALTER TABLE ice_t REPLACE COLUMNS (
+ id INT,
+ name STRING DEFAULT 'unknown1',
+ age INT DEFAULT 25);
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
new file mode 100644
index 00000000000..4ec6abf4faf
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_alter_default_column.q
@@ -0,0 +1,49 @@
+CREATE TABLE ice_t (
+ id INT)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+
+INSERT INTO ice_t (id) VALUES (1);
+
+ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ score DECIMAL(5,2) DEFAULT 100.00,
+ category STRING DEFAULT 'general');
+
+INSERT INTO ice_t (id) VALUES (2);
+
+SELECT * FROM ice_t ORDER BY id;
+
+ALTER TABLE ice_t REPLACE COLUMNS (id INT,
+ point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ category STRING DEFAULT 'general');
+
+SELECT * FROM ice_t ORDER BY id;
+
+-- change default of a field of Struct column
+ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT, y:INT> DEFAULT
'x:100,y:88';
+
+-- rename and change default value of age column
+ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21;
+
+INSERT INTO ice_t (id) VALUES (3);
+
+SELECT * FROM ice_t ORDER BY id;
+
+-- Rename the struct column with default changes
+ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT, y:INT> DEFAULT
'x:55,y:88';
+
+INSERT INTO ice_t (id) VALUES (4);
+
+SELECT * FROM ice_t ORDER BY id;
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out
b/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out
new file mode 100644
index 00000000000..0297b5f65ef
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/negative/iceberg_replace_column_with_default_change.q.out
@@ -0,0 +1,30 @@
+PREHOOK: query: CREATE TABLE ice_t (
+ id INT,
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25
+)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: CREATE TABLE ice_t (
+ id INT,
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25
+)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (
+ id INT,
+ name STRING DEFAULT 'unknown1',
+ age INT DEFAULT 25)
+PREHOOK: type: ALTERTABLE_REPLACECOLS
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+FAILED: Execution Error, return code 40013 from
org.apache.hadoop.hive.ql.ddl.DDLTask. Unable to alter table.
MetaException(message:Unsupported operation to use REPLACE COLUMNS for adding a
column, changing a column type, column comment, column default or reordering
columns. Only use REPLACE COLUMNS for dropping columns. For the other
operations, consider using the ADD COLUMNS or CHANGE COLUMN commands.)
+#### A masked pattern was here ####
+
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
new file mode 100644
index 00000000000..bf7cdcf4d96
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_alter_default_column.q.out
@@ -0,0 +1,161 @@
+PREHOOK: query: CREATE TABLE ice_t (
+ id INT)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: CREATE TABLE ice_t (
+ id INT)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: INSERT INTO ice_t (id) VALUES (1)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: INSERT INTO ice_t (id) VALUES (1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ score DECIMAL(5,2) DEFAULT 100.00,
+ category STRING DEFAULT 'general')
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: ALTER TABLE ice_t ADD COLUMNS (point STRUCT<x:INT, y:INT>
DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ score DECIMAL(5,2) DEFAULT 100.00,
+ category STRING DEFAULT 'general')
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: INSERT INTO ice_t (id) VALUES (2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: INSERT INTO ice_t (id) VALUES (2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: SELECT * FROM ice_t ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM ice_t ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL NULL NULL NULL NULL NULL NULL NULL
+2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 100.00 general
+PREHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT,
+ point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ category STRING DEFAULT 'general')
+PREHOOK: type: ALTERTABLE_REPLACECOLS
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: ALTER TABLE ice_t REPLACE COLUMNS (id INT,
+ point STRUCT<x:INT, y:INT> DEFAULT 'x:100,y:99',
+ name STRING DEFAULT 'unknown',
+ age INT DEFAULT 25,
+ salary DOUBLE DEFAULT 50000.0,
+ is_active BOOLEAN DEFAULT TRUE,
+ created_date DATE DEFAULT '2024-01-01',
+ created_ts TIMESTAMP DEFAULT '2024-01-01T10:00:00',
+ category STRING DEFAULT 'general')
+POSTHOOK: type: ALTERTABLE_REPLACECOLS
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: SELECT * FROM ice_t ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM ice_t ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL NULL NULL NULL NULL NULL NULL
+2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
+PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point STRUCT<x:INT,
y:INT> DEFAULT 'x:100,y:88'
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN age age_new int DEFAULT 21
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: INSERT INTO ice_t (id) VALUES (3)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: INSERT INTO ice_t (id) VALUES (3)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: SELECT * FROM ice_t ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM ice_t ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL NULL NULL NULL NULL NULL NULL
+2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
+3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
+PREHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT 'x:55,y:88'
+PREHOOK: type: ALTERTABLE_RENAMECOL
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: ALTER TABLE ice_t CHANGE COLUMN point point_new STRUCT<x:INT,
y:INT> DEFAULT 'x:55,y:88'
+POSTHOOK: type: ALTERTABLE_RENAMECOL
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: INSERT INTO ice_t (id) VALUES (4)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_t
+POSTHOOK: query: INSERT INTO ice_t (id) VALUES (4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_t
+PREHOOK: query: SELECT * FROM ice_t ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_t
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM ice_t ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_t
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL NULL NULL NULL NULL NULL NULL
+2 {"x":100,"y":99} unknown 25 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
+3 {"x":100,"y":88} unknown 21 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
+4 {"x":55,"y":88} unknown 21 50000.0 true 2024-01-01
2024-01-01 10:00:00 general
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index ddce6aa85af..8d4e943052e 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -2286,12 +2286,17 @@ columnRefOrder
columnNameType
@init { pushMsg("column specification", state); }
@after { popMsg(state); }
- : colName=identifier colType (KW_COMMENT comment=StringLiteral)?
- -> {containExcludedCharForCreateTableColumnName($colName.text)}?
{throwColumnNameException()}
- -> {$comment == null}? ^(TOK_TABCOL $colName colType)
- -> ^(TOK_TABCOL $colName colType $comment)
+ : colName=identifier colType
+ (KW_COMMENT comment=StringLiteral)?
+ defaultClause
+ -> ^(TOK_TABCOL $colName colType $comment? defaultClause?)
;
+defaultClause
+ : (KW_DEFAULT v=expression -> ^(TOK_DEFAULT_VALUE $v))?
+ ;
+
+
columnNameTypeOrConstraint
@init { pushMsg("column name or constraint", state); }
@after { popMsg(state); }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java
index 3ee8d74cea9..c9b74eca9d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLUtils.java
@@ -24,6 +24,7 @@
import java.util.Optional;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -31,7 +32,9 @@
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.hooks.Entity.Type;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -248,4 +251,14 @@ public static boolean hasTransformsInPartitionSpec(Table
table) {
table.getStorageHandler().getPartitionTransformSpec(table).stream()
.anyMatch(spec -> spec.getTransformType() !=
TransformSpec.TransformType.IDENTITY);
}
+
+ public static void setDefaultColumnValues(Table table, TableName tableName,
+ List<ConstraintsUtils.ConstraintInfo> defaultConstraintsInfo,
Configuration conf) throws SemanticException {
+ boolean isNativeColumnDefaultSupported = table.getStorageHandler() != null
&& table.getStorageHandler()
+ .supportsDefaultColumnValues(table.getParameters());
+ List<SQLDefaultConstraint> defaultConstraints = new ArrayList<>();
+ ConstraintsUtils.constraintInfosToDefaultConstraints(tableName,
defaultConstraintsInfo, defaultConstraints,
+ isNativeColumnDefaultSupported);
+ SessionStateUtil.addResourceOrThrow(conf,
SessionStateUtil.COLUMN_DEFAULTS, defaultConstraints);
+ }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java
index 4b490132165..c4d6a75876e 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/add/AlterTableAddColumnsAnalyzer.java
@@ -18,15 +18,18 @@
package org.apache.hadoop.hive.ql.ddl.table.column.add;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -46,7 +49,10 @@ public AlterTableAddColumnsAnalyzer(QueryState queryState)
throws SemanticExcept
@Override
protected void analyzeCommand(TableName tableName, Map<String, String>
partitionSpec, ASTNode command)
throws SemanticException {
- List<FieldSchema> newCols = getColumns((ASTNode) command.getChild(0));
+ List<ConstraintsUtils.ConstraintInfo> defaultConstraintsInfo = new
ArrayList<>();
+ List<FieldSchema> newCols =
+ getColumns((ASTNode) command.getChild(0), true,
ctx.getTokenRewriteStream(), new ArrayList<>(), new ArrayList<>(), new
ArrayList<>(),
+ new ArrayList<>(), defaultConstraintsInfo, new ArrayList<>(),
conf);
boolean isCascade = false;
if (null != command.getFirstChildWithType(HiveParser.TOK_CASCADE)) {
isCascade = true;
@@ -59,6 +65,8 @@ protected void analyzeCommand(TableName tableName,
Map<String, String> partition
}
addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(),
false);
+
+ DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintsInfo,
conf);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
desc)));
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java
index cf017895f44..d3f34cc62c9 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/change/AlterTableChangeColumnAnalyzer.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.hive.metastore.api.SkewedInfo;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
@@ -118,6 +119,7 @@ private Constraints getConstraints(TableName tableName,
ASTNode command, String
List<SQLUniqueConstraint> uniqueConstraints = null;
List<SQLNotNullConstraint> notNullConstraints = null;
List<SQLDefaultConstraint> defaultConstraints = null;
+ List<ConstraintsUtils.ConstraintInfo> defaultConstraintInfo = null;
List<SQLCheckConstraint> checkConstraints = null;
if (constraintChild != null) {
// Process column constraint
@@ -128,10 +130,9 @@ private Constraints getConstraints(TableName tableName,
ASTNode command, String
checkConstraints, (ASTNode) command.getChild(2),
this.ctx.getTokenRewriteStream());
break;
case HiveParser.TOK_DEFAULT_VALUE:
- defaultConstraints = new ArrayList<>();
- ConstraintsUtils.constraintInfosToDefaultConstraints(tableName,
+ defaultConstraintInfo =
ConstraintsUtils.processDefaultConstraints(constraintChild,
ImmutableList.of(newColumnName),
- (ASTNode) command.getChild(2),
this.ctx.getTokenRewriteStream()), defaultConstraints, false);
+ (ASTNode) command.getChild(2),
this.ctx.getTokenRewriteStream());
break;
case HiveParser.TOK_NOT_NULL:
notNullConstraints = new ArrayList<>();
@@ -162,8 +163,18 @@ private Constraints getConstraints(TableName tableName,
ASTNode command, String
ConstraintsUtils.validateCheckConstraint(table.getCols(),
checkConstraints, ctx.getConf());
}
- if (table.getTableType() == TableType.EXTERNAL_TABLE &&
- ConstraintsUtils.hasEnabledOrValidatedConstraints(notNullConstraints,
defaultConstraints, checkConstraints)) {
+ boolean isNativeColumnDefaultSupported = table.getStorageHandler() != null
&& table.getStorageHandler()
+ .supportsDefaultColumnValues(table.getParameters());
+
+ if (defaultConstraintInfo != null) {
+ defaultConstraints = new ArrayList<>();
+ ConstraintsUtils.constraintInfosToDefaultConstraints(tableName,
defaultConstraintInfo, defaultConstraints,
+ isNativeColumnDefaultSupported);
+ DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintInfo,
ctx.getConf());
+ }
+
+ if (table.getTableType() == TableType.EXTERNAL_TABLE &&
ConstraintsUtils.hasEnabledOrValidatedConstraints(
+ notNullConstraints, defaultConstraints, checkConstraints,
isNativeColumnDefaultSupported)) {
throw new SemanticException(ErrorMsg.INVALID_CSTR_SYNTAX.getMsg(
"Constraints are disallowed with External tables. Only RELY is
allowed."));
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java
index 933b80e5393..99a519c2073 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/replace/AlterTableReplaceColumnsAnalyzer.java
@@ -18,15 +18,18 @@
package org.apache.hadoop.hive.ql.ddl.table.column.replace;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLUtils;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -46,7 +49,10 @@ public AlterTableReplaceColumnsAnalyzer(QueryState
queryState) throws SemanticEx
@Override
protected void analyzeCommand(TableName tableName, Map<String, String>
partitionSpec, ASTNode command)
throws SemanticException {
- List<FieldSchema> newCols = getColumns((ASTNode) command.getChild(0));
+ List<ConstraintsUtils.ConstraintInfo> defaultConstraintsInfo = new
ArrayList<>();
+ List<FieldSchema> newCols =
+ getColumns((ASTNode) command.getChild(0), true,
ctx.getTokenRewriteStream(), new ArrayList<>(),
+ new ArrayList<>(), new ArrayList<>(), new ArrayList<>(),
defaultConstraintsInfo, new ArrayList<>(), conf);
boolean isCascade = false;
if (null != command.getFirstChildWithType(HiveParser.TOK_CASCADE)) {
isCascade = true;
@@ -59,6 +65,7 @@ protected void analyzeCommand(TableName tableName,
Map<String, String> partition
}
addInputsOutputsAlterTable(tableName, partitionSpec, desc, desc.getType(),
false);
+ DDLUtils.setDefaultColumnValues(table, tableName, defaultConstraintsInfo,
conf);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
desc)));
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java
index 0d6950b7f62..daac11c1858 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constraint/ConstraintsUtils.java
@@ -525,7 +525,8 @@ private static void validateCheckExpr(ExprNodeDesc
checkExpr) throws SemanticExc
}
public static boolean
hasEnabledOrValidatedConstraints(List<SQLNotNullConstraint> notNullConstraints,
- List<SQLDefaultConstraint> defaultConstraints, List<SQLCheckConstraint>
checkConstraints) {
+ List<SQLDefaultConstraint> defaultConstraints, List<SQLCheckConstraint>
checkConstraints,
+ boolean isNativeColumnDefaultSupported) {
if (notNullConstraints != null) {
for (SQLNotNullConstraint nnC : notNullConstraints) {
if (nnC.isEnable_cstr() || nnC.isValidate_cstr()) {
@@ -534,7 +535,7 @@ public static boolean
hasEnabledOrValidatedConstraints(List<SQLNotNullConstraint
}
}
- if (defaultConstraints != null && !defaultConstraints.isEmpty()) {
+ if (defaultConstraints != null && !defaultConstraints.isEmpty() &&
!isNativeColumnDefaultSupported) {
return true;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java
index 37858b8af0c..28426406c8d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/create/CreateTableAnalyzer.java
@@ -655,7 +655,7 @@ ASTNode analyzeCreateTable(ASTNode ast, QB qb,
PlannerContext plannerCtx)
SessionStateUtil.addResourceOrThrow(conf, META_TABLE_LOCATION,
tblLocation);
if (isExt &&
ConstraintsUtils.hasEnabledOrValidatedConstraints(notNullConstraints,
crtTblDesc.getDefaultConstraints(),
- checkConstraints)) {
+ checkConstraints, isNativeColumnDefaultSupported)) {
throw new SemanticException(ErrorMsg.INVALID_CSTR_SYNTAX.getMsg(
"Constraints are disallowed with External tables. " + "Only RELY
is allowed."));
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index f0850a27be6..f3415219d26 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -95,7 +95,6 @@
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
-import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -918,8 +917,6 @@ public static List<FieldSchema> getColumns(
ASTNode typeChild = (ASTNode) (child.getChild(1));
col.setType(getTypeStringFromAST(typeChild));
- // child 2 is the optional comment of the column
- // child 3 is the optional constraint
ASTNode constraintChild = null;
if (child.getChildCount() == 4) {
col.setComment(unescapeSQLString(child.getChild(2).getText()));
@@ -931,8 +928,9 @@ public static List<FieldSchema> getColumns(
constraintChild = (ASTNode) child.getChild(2);
}
if (constraintChild != null) {
- final TableName tName =
- getQualifiedTableName((ASTNode) parent.getChild(0),
MetaStoreUtils.getDefaultCatalog(conf));
+ final TableName tName = constraintChild.getToken().getType() !=
HiveParser.TOK_DEFAULT_VALUE ?
+ getQualifiedTableName((ASTNode) parent.getChild(0),
MetaStoreUtils.getDefaultCatalog(conf)) :
+ null;
// TODO CAT - for now always use the default catalog. Eventually
will want to see if
// the user specified a catalog
// Process column constraint
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
index 32f38bc0126..d1367056855 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionStateUtil.java
@@ -39,6 +39,7 @@ public class SessionStateUtil {
private static final String CONFLICT_DETECTION_FILTER =
"conflictDetectionFilter.";
public static final String DEFAULT_TABLE_LOCATION = "defaultLocation";
public static final String MISSING_COLUMNS = "missingColumns";
+ public static final String COLUMN_DEFAULTS = "columnDefaults";
private SessionStateUtil() {
}
