szehon-ho commented on code in PR #55836: URL: https://github.com/apache/spark/pull/55836#discussion_r3262005763
########## sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/ChangeArgs.scala: ########## @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.pipelines.autocdc + +import java.util.Locale + +import org.apache.spark.sql.{AnalysisException, Column} +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser +import org.apache.spark.sql.catalyst.util.QuotingUtils +import org.apache.spark.sql.types.StructType + +/** + * A single, unqualified column identifier (no nested path or table/alias qualifier). Backticks + * are consumed: "`a.b`" is stored as "a.b" in [[name]]. Use [[name]] for direct schema-fieldName + * comparison and [[quoted]] for APIs that re-parse identifier strings. + */ +case class UnqualifiedColumnName private (name: String) { + def quoted: String = QuotingUtils.quoteIdentifier(name) +} + +object UnqualifiedColumnName { + def apply(input: String): UnqualifiedColumnName = { + val nameParts = CatalystSqlParser.parseMultipartIdentifier(input) + if (nameParts.length != 1) { + throw multipartColumnIdentifierError(input, nameParts) + } + new UnqualifiedColumnName(nameParts.head) + } + + private def multipartColumnIdentifierError( + columnName: String, + nameParts: Seq[String] + ): AnalysisException = + new AnalysisException( + errorClass = "AUTOCDC_MULTIPART_COLUMN_IDENTIFIER", + messageParameters = Map( + "columnName" -> columnName, + "nameParts" -> nameParts.mkString(", ") + ) + ) +} + +sealed trait ColumnSelection +object ColumnSelection { + + case class IncludeColumns(columns: Seq[UnqualifiedColumnName]) extends ColumnSelection + case class ExcludeColumns(columns: Seq[UnqualifiedColumnName]) + extends ColumnSelection + + /** + * Applies [[ColumnSelection]] to a [[StructType]] and returns the filtered schema. Field + * order follows the original schema; filtering happens in place. + */ + def applyToSchema( + schemaName: String, + schema: StructType, + columnSelection: Option[ColumnSelection], + ignoreCase: Boolean): StructType = columnSelection match { + case None => + // A none column selection is interpreted as a no-op. + schema + case Some(IncludeColumns(cols)) => + val includeColumnNames = cols.map(_.name) + validateColumnsExistInSchema(schemaName, schema, includeColumnNames, ignoreCase) + + val caseNormalizedIncludeColumnNames = + includeColumnNames.map(normalizeCase(_, ignoreCase)).toSet + + StructType( + schema.fields.filter(schemaField => + caseNormalizedIncludeColumnNames.contains(normalizeCase(schemaField.name, ignoreCase)) + ) + ) + case Some(ExcludeColumns(cols)) => + val excludeColumnNames = cols.map(_.name) + validateColumnsExistInSchema(schemaName, schema, excludeColumnNames, ignoreCase) + + val caseNormalizedExcludeColumnNames = + excludeColumnNames.map(normalizeCase(_, ignoreCase)).toSet + + StructType( + schema.fields.filterNot(schemaField => + caseNormalizedExcludeColumnNames.contains(normalizeCase(schemaField.name, ignoreCase)) + ) + ) + } + + private def validateColumnsExistInSchema( + schemaName: String, + schema: StructType, + columnNames: Seq[String], + ignoreCase: Boolean): Unit = { + val caseNormalizedSchemaColumns = + schema.fieldNames.map(normalizeCase(_, ignoreCase)).toSet + + // Compare folded forms but report the missing and available columns using their original + // casing so error messages reflect what the user actually wrote and what the schema holds. + val columnsMissingInSchema = columnNames + .filterNot(columnName => + caseNormalizedSchemaColumns.contains(normalizeCase(columnName, ignoreCase)) + ) + .distinct + + if (columnsMissingInSchema.nonEmpty) { + throw new AnalysisException( + errorClass = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA", + messageParameters = Map( + "caseSensitivity" -> CaseSensitivityLabels.of(ignoreCase), + "schemaName" -> schemaName, + "missingColumns" -> columnsMissingInSchema.mkString(", "), + "availableColumns" -> schema.fieldNames.mkString(", ") + )) + } + } + + /** + * If ignoreCase, normalize all strings to lowercase for stable comparison. + */ + private def normalizeCase(name: String, ignoreCase: Boolean): String = { Review Comment: Consider using Spark's built-in case-sensitivity machinery here instead of a custom `normalizeCase` + `Set` fold: - **`Resolver`** (`caseSensitiveResolution` / `caseInsensitiveResolution` from `org.apache.spark.sql.catalyst.analysis`) for comparing two identifier strings — same pattern as `SchemaUtils.findColumnPosition` and the analyzer. When this is wired to a session later, `spark.sessionState.conf.resolver` keeps behavior aligned with `spark.sql.caseSensitive`. - **`StructType.getFieldIndex` / `getFieldIndexCaseInsensitive`** for top-level schema membership — matches what `StructType` already uses internally (`CaseInsensitiveMap` + `Locale.ROOT`), e.g. the split in `StaxXmlParser`. You can keep `ignoreCase: Boolean` on the public API for testability and map it internally to a `Resolver` or the `getFieldIndex*` helpers. That avoids maintaining a parallel normalization scheme. (Minor note: `Resolver` uses `equalsIgnoreCase` while `getFieldIndexCaseInsensitive` uses `Locale.ROOT` lowercasing — they can diverge for unusual Unicode; for schema field lookup, `getFieldIndexCaseInsensitive` is usually the better match.) Also, please add a small unit test locking in the empty-list semantics we discussed: `None` vs `Some(IncludeColumns(Seq()))` (empty `StructType`) vs `Some(ExcludeColumns(Seq()))` (no-op). A few lines in `ChangeArgsSuite` would make that contract clear for future readers. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
