This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-0.6 in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 3d9686d76147a83b4b65dfc1ff83da14b00ed409 Author: Kerwin <[email protected]> AuthorDate: Tue Jan 9 10:43:26 2024 +0800 [hive] Fix hive to be compatible with uppercase fields when using predicates to query paimon external tables. (#2659) --- .../hive/SearchArgumentToPredicateConverter.java | 20 +++++++++++++------- .../java/org/apache/paimon/hive/utils/HiveUtils.java | 6 +++--- .../java/org/apache/paimon/hive/HiveReadITCase.java | 7 +++++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java index 67d74243b..fa7881771 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/SearchArgumentToPredicateConverter.java @@ -41,7 +41,7 @@ import java.util.stream.Collectors; import static org.apache.paimon.predicate.PredicateBuilder.convertJavaObject; -/** Converts {@link SearchArgument} to {@link Predicate} with best effort. */ +/** Converts {@link SearchArgument} to {@link Predicate} with the best effort. */ public class SearchArgumentToPredicateConverter { private static final Logger LOG = @@ -55,20 +55,26 @@ public class SearchArgumentToPredicateConverter { private final PredicateBuilder builder; public SearchArgumentToPredicateConverter( - SearchArgument sarg, + SearchArgument searchArgument, List<String> columnNames, List<DataType> columnTypes, @Nullable Set<String> readColumnNames) { - this.root = sarg.getExpression(); - this.leaves = sarg.getLeaves(); - this.columnNames = columnNames; + this.root = searchArgument.getExpression(); + this.leaves = searchArgument.getLeaves(); + this.columnNames = + columnNames.stream().map(String::toLowerCase).collect(Collectors.toList()); this.columnTypes = columnTypes; + if (readColumnNames != null) { + readColumnNames = + readColumnNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + } this.readColumnNames = readColumnNames; + this.builder = new PredicateBuilder( RowType.of( - columnTypes.toArray(new DataType[0]), - columnNames.toArray(new String[0]))); + this.columnTypes.toArray(new DataType[0]), + this.columnNames.toArray(new String[0]))); } public Optional<Predicate> convert() { diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java index 20b7cd017..e2959ea6a 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/utils/HiveUtils.java @@ -57,8 +57,8 @@ public class HiveUtils { public static Optional<Predicate> createPredicate( TableSchema tableSchema, JobConf jobConf, boolean limitToReadColumnNames) { - SearchArgument sarg = ConvertAstToSearchArg.createFromConf(jobConf); - if (sarg == null) { + SearchArgument searchArgument = ConvertAstToSearchArg.createFromConf(jobConf); + if (searchArgument == null) { return Optional.empty(); } Set<String> readColumnNames = null; @@ -79,7 +79,7 @@ public class HiveUtils { } SearchArgumentToPredicateConverter converter = new SearchArgumentToPredicateConverter( - sarg, + searchArgument, tableSchema.fieldNames(), tableSchema.logicalRowType().getFieldTypes(), readColumnNames); diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java index bdeff0acd..8e4ea57c3 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java @@ -91,5 +91,12 @@ public class HiveReadITCase extends HiveTestBase { assertThat(result).containsExactly("Hello", "Paimon"); result = hiveShell.executeQuery("SELECT Col2 FROM " + tableName); assertThat(result).containsExactly("Hello", "Paimon"); + + result = hiveShell.executeQuery("SELECT * FROM " + tableName + " WHERE col2 = 'Hello'"); + assertThat(result).containsExactly("1\tHello"); + result = + hiveShell.executeQuery( + "SELECT * FROM " + tableName + " WHERE Col2 in ('Hello', 'Paimon')"); + assertThat(result).containsExactly("1\tHello", "2\tPaimon"); } }
