This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-0.6 in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit 770334b3e5f106443c8d9a6be3c645b8b2185c97 Author: Kerwin <[email protected]> AuthorDate: Thu Jan 4 22:07:14 2024 +0800 [hive] Compatible with upper and lower case when use hivesql queries paimon external table (#2639) --- .../paimon/hive/mapred/PaimonRecordReader.java | 19 ++++- .../org/apache/paimon/hive/HiveReadITCase.java | 95 ++++++++++++++++++++++ .../org/apache/paimon/hive/HiveWriteITCase.java | 28 +------ 3 files changed, 114 insertions(+), 28 deletions(-) diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java index 4e30b83c7..64de2c048 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/mapred/PaimonRecordReader.java @@ -33,7 +33,11 @@ import org.apache.hadoop.mapred.RecordReader; import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * Base {@link RecordReader} for paimon. Reads {@link KeyValue}s from data files and picks out @@ -67,9 +71,20 @@ public class PaimonRecordReader implements RecordReader<Void, RowDataContainer> List<String> selectedColumns, @Nullable String tagToPartField) throws IOException { - if (!paimonColumns.equals(selectedColumns)) { + + LinkedHashMap<String, Integer> paimonColumnIndexMap = + IntStream.range(0, paimonColumns.size()) + .boxed() + .collect( + Collectors.toMap( + index -> paimonColumns.get(index).toLowerCase(), + index -> index, + (existing, replacement) -> existing, + LinkedHashMap::new)); + + if (!new ArrayList<>(paimonColumnIndexMap.keySet()).equals(selectedColumns)) { readBuilder.withProjection( - selectedColumns.stream().mapToInt(paimonColumns::indexOf).toArray()); + selectedColumns.stream().mapToInt(paimonColumnIndexMap::get).toArray()); } if (hiveColumns.equals(selectedColumns)) { diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java new file mode 100644 index 000000000..bdeff0acd --- /dev/null +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCase.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.catalog.AbstractCatalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.hive.mapred.PaimonInputFormat; +import org.apache.paimon.hive.mapred.PaimonRecordReader; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; + +import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; + +/** IT cases for {@link PaimonRecordReader} and {@link PaimonInputFormat}. */ +public class HiveReadITCase extends HiveTestBase { + + @Test + public void testReExternalTableWithIgnoreCase() throws Exception { + // Create hive external table with paimon table + String tableName = "with_ignore_case"; + + // Create a paimon table + Schema schema = + new Schema( + Lists.newArrayList( + new DataField(0, "col1", DataTypes.INT(), "first comment"), + new DataField(1, "Col2", DataTypes.STRING(), "second comment")), + Collections.emptyList(), + Collections.emptyList(), + Maps.newHashMap(), + ""); + Identifier identifier = Identifier.create(DATABASE_TEST, tableName); + Path tablePath = AbstractCatalog.newTableLocation(path, identifier); + new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + + // Create hive external table + String hiveSql = + String.join( + "\n", + Arrays.asList( + "CREATE EXTERNAL TABLE " + tableName + " ", + "STORED BY '" + PaimonStorageHandler.class.getName() + "'", + "LOCATION '" + tablePath.toUri().toString() + "'")); + assertThatCode(() -> hiveShell.execute(hiveSql)).doesNotThrowAnyException(); + + List<String> result = hiveShell.executeQuery("SHOW CREATE TABLE " + tableName); + assertThat(result) + .containsAnyOf( + "CREATE EXTERNAL TABLE `with_paimon_table`(", + " `col1` int COMMENT 'first comment', ", + " `col2` string COMMENT 'second comment')", + "ROW FORMAT SERDE ", + " 'org.apache.paimon.hive.PaimonSerDe' ", + "STORED BY ", + " 'org.apache.paimon.hive.PaimonStorageHandler' "); + + hiveShell.execute("INSERT INTO " + tableName + " VALUES (1,'Hello'),(2,'Paimon')"); + result = hiveShell.executeQuery("SELECT col2, col1 FROM " + tableName); + assertThat(result).containsExactly("Hello\t1", "Paimon\t2"); + result = hiveShell.executeQuery("SELECT col2 FROM " + tableName); + assertThat(result).containsExactly("Hello", "Paimon"); + result = hiveShell.executeQuery("SELECT Col2 FROM " + tableName); + assertThat(result).containsExactly("Hello", "Paimon"); + } +} diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java index f50e977d5..d057cb52d 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java @@ -28,7 +28,6 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.hive.mapred.PaimonOutputFormat; import org.apache.paimon.hive.objectinspector.PaimonObjectInspectorFactory; -import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.table.Table; @@ -44,20 +43,14 @@ import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.StringUtils; -import com.klarna.hiverunner.HiveShell; -import com.klarna.hiverunner.annotations.HiveSQL; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector; -import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; import java.util.ArrayList; import java.util.Arrays; @@ -74,17 +67,10 @@ import static org.apache.paimon.hive.RandomGenericRowDataGenerator.randomBigDeci import static org.assertj.core.api.Assertions.assertThat; /** IT cases for {@link PaimonStorageHandler} and {@link PaimonOutputFormat}. */ -@RunWith(PaimonEmbeddedHiveRunner.class) -public class HiveWriteITCase { - - @ClassRule public static TemporaryFolder folder = new TemporaryFolder(); - - @HiveSQL(files = {}) - private static HiveShell hiveShell; +public class HiveWriteITCase extends HiveTestBase { private static String engine; - private String commitUser; private long commitIdentifier; @BeforeClass @@ -95,20 +81,10 @@ public class HiveWriteITCase { @Before public void before() { - hiveShell.execute("SET hive.execution.engine=mr"); - - hiveShell.execute("CREATE DATABASE IF NOT EXISTS test_db"); - hiveShell.execute("USE test_db"); - - commitUser = UUID.randomUUID().toString(); + hiveShell.execute("SET hive.execution.engine=" + engine); commitIdentifier = 0; } - @After - public void after() { - hiveShell.execute("DROP DATABASE IF EXISTS test_db CASCADE"); - } - private String createChangelogExternalTable( RowType rowType, List<String> partitionKeys,
