fmorillo7694 commented on code in PR #206: URL: https://github.com/apache/flink-connector-aws/pull/206#discussion_r4125198029
########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( + ResolvedSchema resolvedSchema, Map<String, String> targetParameters) { + List<Column> columns = resolvedSchema.getColumns(); + List<String> notNullColumns = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + if (column instanceof Column.PhysicalColumn) { + if (!column.getDataType().getLogicalType().isNullable()) { + notNullColumns.add(column.getName()); + } + // Glue's Hive-style type strings are lossy for some types (for example + // TIMESTAMP(3) has no Glue representation and would come back as + // TIMESTAMP(6)). When the Glue round-trip does not reproduce the declared + // type, record the original so the read path can restore it exactly. + LogicalType declared = column.getDataType().getLogicalType(); + LogicalType glueRoundTrip = + TYPE_CONVERTER + .toFlinkDataType( + TYPE_CONVERTER.toGlueDataType(column.getDataType())) + .getLogicalType(); + if (!declared.copy(true).equals(glueRoundTrip.copy(true))) { + targetParameters.put( + COLUMN_PREFIX + i + PHYSICAL_DATA_TYPE_SUFFIX, + declared.asSerializableString()); + } + } + if (column instanceof Column.ComputedColumn) { + Column.ComputedColumn computed = (Column.ComputedColumn) column; + targetParameters.put(COLUMN_PREFIX + i + COMPUTED_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + COMPUTED_EXPR_SUFFIX, + serializeExpression(column.getName(), computed.getExpression())); + } else if (column instanceof Column.MetadataColumn) { + Column.MetadataColumn metadata = (Column.MetadataColumn) column; + final int position = i; + targetParameters.put(COLUMN_PREFIX + i + METADATA_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_DATA_TYPE_SUFFIX, + column.getDataType().getLogicalType().asSerializableString()); + metadata.getMetadataKey() + .ifPresent( + key -> + targetParameters.put( + COLUMN_PREFIX + position + METADATA_KEY_SUFFIX, + key)); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_VIRTUAL_SUFFIX, + String.valueOf(metadata.isVirtual())); + } + } + + if (!notNullColumns.isEmpty()) { + targetParameters.put(NOT_NULL_COLUMNS, String.join(",", notNullColumns)); + } + + List<WatermarkSpec> watermarkSpecs = resolvedSchema.getWatermarkSpecs(); + for (int i = 0; i < watermarkSpecs.size(); i++) { + WatermarkSpec spec = watermarkSpecs.get(i); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_ROWTIME_SUFFIX, spec.getRowtimeAttribute()); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_STRATEGY_EXPR_SUFFIX, + serializeExpression(spec.getRowtimeAttribute(), spec.getWatermarkExpression())); + } + + resolvedSchema + .getPrimaryKey() + .ifPresent( + primaryKey -> { + targetParameters.put(PRIMARY_KEY_NAME, primaryKey.getName()); + targetParameters.put( + PRIMARY_KEY_COLUMNS, String.join(",", primaryKey.getColumns())); + }); + } + + /** + * Returns whether the given Glue table parameter key was written by {@link + * #serializeNonPhysicalSchema} and must therefore be hidden from the table options exposed to + * users. + * + * @param parameterKey the Glue table parameter key + * @return true when the key is an internal schema-fidelity parameter + */ + public static boolean isSchemaParameter(String parameterKey) { + return parameterKey.startsWith(SCHEMA_PARAMETER_PREFIX); + } + + /** + * Restores computed and metadata columns from the Glue table parameters, merging them with the + * physical columns at their originally declared positions, then re-applies watermarks and the + * primary key. + * + * @param parameters the Glue table parameters (may be null) + * @param physicalColumnAdders ordered adders for the physical columns read from Glue + * @param schemaBuilder the schema builder to populate + */ + public static void applySchemaWithNonPhysicalColumns( + Map<String, String> parameters, + List<PhysicalColumnSpec> physicalColumns, + Schema.Builder schemaBuilder) { + + // Rebuild non-physical columns keyed by their original position in the schema. + TreeMap<Integer, Consumer<Schema.Builder>> nonPhysicalByPosition = new TreeMap<>(); + if (parameters != null) { + for (Map.Entry<String, String> entry : parameters.entrySet()) { + String key = entry.getKey(); + if (key.startsWith(COLUMN_PREFIX) && key.endsWith(COMPUTED_NAME_SUFFIX)) { Review Comment: Yes — `GlueCatalog.rejectReservedOptions` now throws a `CatalogException` for any user table option starting with `flink.schema.` or equal to `flink.original-table-name` / `flink.original-database-name` / `flink.original-partition-keys`, on create and alter. `GlueCatalogSchemaFidelityTest.testReservedOptionsAreRejected` covers it. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( Review Comment: Adopted your shape (`serializeSchema` with `LIST_SEPARATOR` and name-keyed parameters) — it also turned out to be the right design for the reordering bug, see the thread on `GlueTableUtils:159`. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( + ResolvedSchema resolvedSchema, Map<String, String> targetParameters) { + List<Column> columns = resolvedSchema.getColumns(); + List<String> notNullColumns = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + if (column instanceof Column.PhysicalColumn) { + if (!column.getDataType().getLogicalType().isNullable()) { + notNullColumns.add(column.getName()); + } + // Glue's Hive-style type strings are lossy for some types (for example + // TIMESTAMP(3) has no Glue representation and would come back as + // TIMESTAMP(6)). When the Glue round-trip does not reproduce the declared + // type, record the original so the read path can restore it exactly. + LogicalType declared = column.getDataType().getLogicalType(); + LogicalType glueRoundTrip = + TYPE_CONVERTER + .toFlinkDataType( + TYPE_CONVERTER.toGlueDataType(column.getDataType())) + .getLogicalType(); + if (!declared.copy(true).equals(glueRoundTrip.copy(true))) { + targetParameters.put( + COLUMN_PREFIX + i + PHYSICAL_DATA_TYPE_SUFFIX, + declared.asSerializableString()); + } + } + if (column instanceof Column.ComputedColumn) { + Column.ComputedColumn computed = (Column.ComputedColumn) column; + targetParameters.put(COLUMN_PREFIX + i + COMPUTED_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + COMPUTED_EXPR_SUFFIX, + serializeExpression(column.getName(), computed.getExpression())); + } else if (column instanceof Column.MetadataColumn) { + Column.MetadataColumn metadata = (Column.MetadataColumn) column; + final int position = i; + targetParameters.put(COLUMN_PREFIX + i + METADATA_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_DATA_TYPE_SUFFIX, + column.getDataType().getLogicalType().asSerializableString()); + metadata.getMetadataKey() + .ifPresent( + key -> + targetParameters.put( + COLUMN_PREFIX + position + METADATA_KEY_SUFFIX, + key)); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_VIRTUAL_SUFFIX, + String.valueOf(metadata.isVirtual())); + } + } + + if (!notNullColumns.isEmpty()) { + targetParameters.put(NOT_NULL_COLUMNS, String.join(",", notNullColumns)); + } + + List<WatermarkSpec> watermarkSpecs = resolvedSchema.getWatermarkSpecs(); + for (int i = 0; i < watermarkSpecs.size(); i++) { + WatermarkSpec spec = watermarkSpecs.get(i); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_ROWTIME_SUFFIX, spec.getRowtimeAttribute()); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_STRATEGY_EXPR_SUFFIX, + serializeExpression(spec.getRowtimeAttribute(), spec.getWatermarkExpression())); + } + + resolvedSchema + .getPrimaryKey() + .ifPresent( + primaryKey -> { + targetParameters.put(PRIMARY_KEY_NAME, primaryKey.getName()); + targetParameters.put( + PRIMARY_KEY_COLUMNS, String.join(",", primaryKey.getColumns())); + }); + } + + /** + * Returns whether the given Glue table parameter key was written by {@link + * #serializeNonPhysicalSchema} and must therefore be hidden from the table options exposed to + * users. + * + * @param parameterKey the Glue table parameter key + * @return true when the key is an internal schema-fidelity parameter + */ + public static boolean isSchemaParameter(String parameterKey) { + return parameterKey.startsWith(SCHEMA_PARAMETER_PREFIX); + } + + /** + * Restores computed and metadata columns from the Glue table parameters, merging them with the + * physical columns at their originally declared positions, then re-applies watermarks and the + * primary key. + * + * @param parameters the Glue table parameters (may be null) + * @param physicalColumnAdders ordered adders for the physical columns read from Glue + * @param schemaBuilder the schema builder to populate + */ + public static void applySchemaWithNonPhysicalColumns( Review Comment: Renamed to `restoreSchema`. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( + ResolvedSchema resolvedSchema, Map<String, String> targetParameters) { + List<Column> columns = resolvedSchema.getColumns(); + List<String> notNullColumns = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + if (column instanceof Column.PhysicalColumn) { + if (!column.getDataType().getLogicalType().isNullable()) { + notNullColumns.add(column.getName()); + } + // Glue's Hive-style type strings are lossy for some types (for example + // TIMESTAMP(3) has no Glue representation and would come back as + // TIMESTAMP(6)). When the Glue round-trip does not reproduce the declared + // type, record the original so the read path can restore it exactly. + LogicalType declared = column.getDataType().getLogicalType(); + LogicalType glueRoundTrip = + TYPE_CONVERTER + .toFlinkDataType( + TYPE_CONVERTER.toGlueDataType(column.getDataType())) + .getLogicalType(); + if (!declared.copy(true).equals(glueRoundTrip.copy(true))) { + targetParameters.put( + COLUMN_PREFIX + i + PHYSICAL_DATA_TYPE_SUFFIX, + declared.asSerializableString()); + } + } + if (column instanceof Column.ComputedColumn) { + Column.ComputedColumn computed = (Column.ComputedColumn) column; + targetParameters.put(COLUMN_PREFIX + i + COMPUTED_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + COMPUTED_EXPR_SUFFIX, + serializeExpression(column.getName(), computed.getExpression())); + } else if (column instanceof Column.MetadataColumn) { + Column.MetadataColumn metadata = (Column.MetadataColumn) column; + final int position = i; + targetParameters.put(COLUMN_PREFIX + i + METADATA_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_DATA_TYPE_SUFFIX, + column.getDataType().getLogicalType().asSerializableString()); + metadata.getMetadataKey() + .ifPresent( + key -> + targetParameters.put( + COLUMN_PREFIX + position + METADATA_KEY_SUFFIX, + key)); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_VIRTUAL_SUFFIX, + String.valueOf(metadata.isVirtual())); + } + } + + if (!notNullColumns.isEmpty()) { + targetParameters.put(NOT_NULL_COLUMNS, String.join(",", notNullColumns)); + } + + List<WatermarkSpec> watermarkSpecs = resolvedSchema.getWatermarkSpecs(); + for (int i = 0; i < watermarkSpecs.size(); i++) { + WatermarkSpec spec = watermarkSpecs.get(i); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_ROWTIME_SUFFIX, spec.getRowtimeAttribute()); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_STRATEGY_EXPR_SUFFIX, + serializeExpression(spec.getRowtimeAttribute(), spec.getWatermarkExpression())); + } + + resolvedSchema + .getPrimaryKey() + .ifPresent( + primaryKey -> { + targetParameters.put(PRIMARY_KEY_NAME, primaryKey.getName()); + targetParameters.put( + PRIMARY_KEY_COLUMNS, String.join(",", primaryKey.getColumns())); + }); + } + + /** + * Returns whether the given Glue table parameter key was written by {@link + * #serializeNonPhysicalSchema} and must therefore be hidden from the table options exposed to + * users. + * + * @param parameterKey the Glue table parameter key + * @return true when the key is an internal schema-fidelity parameter + */ + public static boolean isSchemaParameter(String parameterKey) { + return parameterKey.startsWith(SCHEMA_PARAMETER_PREFIX); + } + + /** + * Restores computed and metadata columns from the Glue table parameters, merging them with the + * physical columns at their originally declared positions, then re-applies watermarks and the + * primary key. + * + * @param parameters the Glue table parameters (may be null) + * @param physicalColumnAdders ordered adders for the physical columns read from Glue + * @param schemaBuilder the schema builder to populate + */ + public static void applySchemaWithNonPhysicalColumns( Review Comment: Adopted — `restoreSchema(parameters, glueColumns, comments, builder)` rebuilds the schema from `flink.schema.column-order`, pulling each column from the non-physical parameters or the `LinkedHashMap<String, DataType>` of Glue columns by name. Columns another engine added to the Glue table afterwards are appended so nothing is silently dropped. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueTableUtils.java: ########## @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.ObjectPath; +import org.apache.flink.table.types.DataType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.glue.model.Column; +import software.amazon.awssdk.services.glue.model.StorageDescriptor; +import software.amazon.awssdk.services.glue.model.Table; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Utility class for working with Glue tables, including transforming Glue-specific metadata into + * Flink-compatible objects. + */ +public class GlueTableUtils { + + /** Logger for logging Glue table operations. */ + private static final Logger LOG = LoggerFactory.getLogger(GlueTableUtils.class); + + /** Glue type converter for type conversions between Flink and Glue types. */ + private final GlueTypeConverter glueTypeConverter; + + /** + * Constructor to initialize GlueTableUtils with a GlueTypeConverter. + * + * @param glueTypeConverter The GlueTypeConverter instance for type mapping. + */ + public GlueTableUtils(GlueTypeConverter glueTypeConverter) { + this.glueTypeConverter = glueTypeConverter; + } + + /** + * Builds a Glue StorageDescriptor from the given table properties, columns, and location. + * + * @param tableProperties Table properties for the Glue table. + * @param glueColumns Columns to be included in the StorageDescriptor. + * @param tableLocation Location of the Glue table. + * @return A newly built StorageDescriptor object. + */ + public StorageDescriptor buildStorageDescriptor( + Map<String, String> tableProperties, List<Column> glueColumns, String tableLocation) { + + return StorageDescriptor.builder().columns(glueColumns).location(tableLocation).build(); + } + + /** + * Extracts the table location based on the table properties and the table path. First, it + * checks for a location key from the connector registry. If no such key is found, it uses a + * default path based on the table path. + * + * @param tableProperties Table properties containing the connector and location. + * @param tablePath The Flink ObjectPath representing the table. + * @return The location of the Glue table. + */ + public String extractTableLocation(Map<String, String> tableProperties, ObjectPath tablePath) { + String connectorType = tableProperties.get("connector"); + if (connectorType != null) { + String locationKey = ConnectorRegistry.getLocationKey(connectorType); + if (locationKey != null && tableProperties.containsKey(locationKey)) { + String location = tableProperties.get(locationKey); + return location; + } + } + + String defaultLocation = + tablePath.getDatabaseName() + "/tables/" + tablePath.getObjectName(); + return defaultLocation; + } + + /** + * Converts a Flink column to a Glue column. The column's data type is converted using the + * GlueTypeConverter. + * + * @param flinkColumn The Flink column to be converted. + * @return The corresponding Glue column. + */ + public Column mapFlinkColumnToGlueColumn(org.apache.flink.table.catalog.Column flinkColumn) { + String glueType = glueTypeConverter.toGlueDataType(flinkColumn.getDataType()); + + // AWS Glue lowercases column names on CreateTable/UpdateTable (verified empirically: + // a column created as "userId" is stored and returned as "userid"). To preserve the + // declared case, store the lowercased name explicitly and stash the original name in + // the "originalName" column parameter, which the read path restores. + String originalName = flinkColumn.getName(); + String glueName = originalName.toLowerCase(); + + Column.Builder builder = Column.builder().name(glueName).type(glueType); + if (!glueName.equals(originalName)) { + builder.parameters( + Collections.singletonMap( + GlueCatalogConstants.ORIGINAL_COLUMN_NAME, originalName)); + } + return builder.build(); + } + + /** + * Converts a Glue table into a Flink schema. Each Glue column is mapped to a Flink column using + * the GlueTypeConverter. Partition columns (stored at the Glue table level, not in the storage + * descriptor) are appended after the data columns so that declared partition keys are part of + * the Flink schema, as required by {@code CatalogTable}. Computed and metadata columns, + * watermarks, and the primary key - which Glue columns cannot represent - are restored from the + * {@code flink.schema.*} table parameters written by {@link GlueFlinkSchemaProperties}. + * + * @param glueTable The Glue table from which the schema will be derived. + * @return A Flink schema constructed from the Glue table's columns. + */ + public Schema getSchemaFromGlueTable(Table glueTable) { Review Comment: Adopted — `getSchemaFromGlueTable` now builds `LinkedHashMap<name, DataType>` (plus a comment map) and delegates to `restoreSchema`. ########## flink-catalog-aws/flink-catalog-aws-glue/src/main/java/org/apache/flink/table/catalog/glue/util/GlueFlinkSchemaProperties.java: ########## @@ -0,0 +1,338 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.catalog.glue.util; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.WatermarkSpec; +import org.apache.flink.table.catalog.exceptions.CatalogException; +import org.apache.flink.table.types.logical.LogicalType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.function.Consumer; + +/** + * Persists the parts of a Flink schema that AWS Glue columns cannot represent - computed columns, + * metadata columns, watermarks, and the primary key - as Glue table parameters, and restores them + * when a table is read back. + * + * <p>Physical columns are stored as native Glue columns (so other engines can read the table), + * while everything else round-trips through {@code flink.schema.*} parameters. Without this, {@code + * CREATE TABLE} statements with watermarks or primary keys would silently lose them on read back, + * and computed or metadata columns would be corrupted into physical columns. + */ +@Internal +public final class GlueFlinkSchemaProperties { + + /** Prefix shared by all schema-fidelity parameters written by this class. */ + public static final String SCHEMA_PARAMETER_PREFIX = "flink.schema."; + + private static final String COLUMN_PREFIX = SCHEMA_PARAMETER_PREFIX + "column."; + private static final String COMPUTED_NAME_SUFFIX = ".computed.name"; + private static final String COMPUTED_EXPR_SUFFIX = ".computed.expr"; + private static final String METADATA_NAME_SUFFIX = ".metadata.name"; + private static final String METADATA_DATA_TYPE_SUFFIX = ".metadata.data-type"; + private static final String METADATA_KEY_SUFFIX = ".metadata.key"; + private static final String METADATA_VIRTUAL_SUFFIX = ".metadata.virtual"; + private static final String PHYSICAL_DATA_TYPE_SUFFIX = ".data-type"; + + private static final String WATERMARK_PREFIX = SCHEMA_PARAMETER_PREFIX + "watermark."; + private static final String WATERMARK_ROWTIME_SUFFIX = ".rowtime"; + private static final String WATERMARK_STRATEGY_EXPR_SUFFIX = ".strategy.expr"; + + private static final String PRIMARY_KEY_NAME = SCHEMA_PARAMETER_PREFIX + "primary-key.name"; + private static final String PRIMARY_KEY_COLUMNS = + SCHEMA_PARAMETER_PREFIX + "primary-key.columns"; + + /** + * Comma-joined names of physical columns declared NOT NULL. Glue type strings carry no + * nullability, so without this the restored primary key would fail schema resolution ("Invalid + * primary key: column is nullable"). + */ + private static final String NOT_NULL_COLUMNS = SCHEMA_PARAMETER_PREFIX + "not-null-columns"; + + private static final GlueTypeConverter TYPE_CONVERTER = new GlueTypeConverter(); + + private GlueFlinkSchemaProperties() {} + + /** + * Serializes the non-physical parts of the given schema into the target parameter map. + * + * @param resolvedSchema the resolved Flink schema + * @param targetParameters the mutable Glue table parameter map to write into + * @throws CatalogException if an expression cannot be serialized to SQL + */ + public static void serializeNonPhysicalSchema( + ResolvedSchema resolvedSchema, Map<String, String> targetParameters) { + List<Column> columns = resolvedSchema.getColumns(); + List<String> notNullColumns = new ArrayList<>(); + for (int i = 0; i < columns.size(); i++) { + Column column = columns.get(i); + if (column instanceof Column.PhysicalColumn) { + if (!column.getDataType().getLogicalType().isNullable()) { + notNullColumns.add(column.getName()); + } + // Glue's Hive-style type strings are lossy for some types (for example + // TIMESTAMP(3) has no Glue representation and would come back as + // TIMESTAMP(6)). When the Glue round-trip does not reproduce the declared + // type, record the original so the read path can restore it exactly. + LogicalType declared = column.getDataType().getLogicalType(); + LogicalType glueRoundTrip = + TYPE_CONVERTER + .toFlinkDataType( + TYPE_CONVERTER.toGlueDataType(column.getDataType())) + .getLogicalType(); + if (!declared.copy(true).equals(glueRoundTrip.copy(true))) { + targetParameters.put( + COLUMN_PREFIX + i + PHYSICAL_DATA_TYPE_SUFFIX, + declared.asSerializableString()); + } + } + if (column instanceof Column.ComputedColumn) { + Column.ComputedColumn computed = (Column.ComputedColumn) column; + targetParameters.put(COLUMN_PREFIX + i + COMPUTED_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + COMPUTED_EXPR_SUFFIX, + serializeExpression(column.getName(), computed.getExpression())); + } else if (column instanceof Column.MetadataColumn) { + Column.MetadataColumn metadata = (Column.MetadataColumn) column; + final int position = i; + targetParameters.put(COLUMN_PREFIX + i + METADATA_NAME_SUFFIX, column.getName()); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_DATA_TYPE_SUFFIX, + column.getDataType().getLogicalType().asSerializableString()); + metadata.getMetadataKey() + .ifPresent( + key -> + targetParameters.put( + COLUMN_PREFIX + position + METADATA_KEY_SUFFIX, + key)); + targetParameters.put( + COLUMN_PREFIX + i + METADATA_VIRTUAL_SUFFIX, + String.valueOf(metadata.isVirtual())); + } + } + + if (!notNullColumns.isEmpty()) { + targetParameters.put(NOT_NULL_COLUMNS, String.join(",", notNullColumns)); + } + + List<WatermarkSpec> watermarkSpecs = resolvedSchema.getWatermarkSpecs(); + for (int i = 0; i < watermarkSpecs.size(); i++) { + WatermarkSpec spec = watermarkSpecs.get(i); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_ROWTIME_SUFFIX, spec.getRowtimeAttribute()); + targetParameters.put( + WATERMARK_PREFIX + i + WATERMARK_STRATEGY_EXPR_SUFFIX, + serializeExpression(spec.getRowtimeAttribute(), spec.getWatermarkExpression())); + } + + resolvedSchema + .getPrimaryKey() + .ifPresent( + primaryKey -> { + targetParameters.put(PRIMARY_KEY_NAME, primaryKey.getName()); + targetParameters.put( + PRIMARY_KEY_COLUMNS, String.join(",", primaryKey.getColumns())); + }); + } + + /** + * Returns whether the given Glue table parameter key was written by {@link + * #serializeNonPhysicalSchema} and must therefore be hidden from the table options exposed to + * users. + * + * @param parameterKey the Glue table parameter key + * @return true when the key is an internal schema-fidelity parameter + */ + public static boolean isSchemaParameter(String parameterKey) { + return parameterKey.startsWith(SCHEMA_PARAMETER_PREFIX); Review Comment: Yes — the `flink.original-*` keys are now rejected as user options (same check as `flink.schema.*`) and filtered out on read. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
