This is an automated email from the ASF dual-hosted git repository.
czweng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 199b302dd [bug] Fix: make hive load the correct FileIO set by conf
(#1371)
199b302dd is described below
commit 199b302dd5c7911b42d1f706ecbcb8a76f7dc639
Author: LsomeYeah <[email protected]>
AuthorDate: Thu Jun 15 17:45:24 2023 +0800
[bug] Fix: make hive load the correct FileIO set by conf (#1371)
This closes #1371.
---
.../java/org/apache/paimon/tests/HiveE2eTest.java | 34 ++++++++++++++++++++++
.../apache/paimon/hive/LocationKeyExtractor.java | 23 +++++++++++++++
.../org/apache/paimon/hive/PaimonMetaHook.java | 7 +++--
3 files changed, 62 insertions(+), 2 deletions(-)
diff --git
a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java
b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java
index 1bcf862f5..79be61211 100644
--- a/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java
+++ b/paimon-e2e-tests/src/test/java/org/apache/paimon/tests/HiveE2eTest.java
@@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledIfSystemProperty;
import org.testcontainers.containers.Container;
+import java.util.Arrays;
import java.util.UUID;
/**
@@ -89,6 +90,39 @@ public class HiveE2eTest extends E2eReaderTestBase {
checkQueryResults(table, this::executeQuery);
}
+ @Test
+ public void testHiveWrite() throws Exception {
+ final String table = "hive_test";
+ String hiveSql =
+ String.join(
+ "\n",
+ Arrays.asList(
+ "CREATE TABLE " + table + " (",
+ "a bigint" + " COMMENT 'The a field',",
+ "b bigint" + " COMMENT 'The b field',",
+ "c string" + " COMMENT 'The c field'",
+ ")",
+ "STORED BY
'org.apache.paimon.hive.PaimonStorageHandler';"));
+ String hql1 = "hiveddl.hql";
+ writeSharedFile(hql1, hiveSql);
+ executeQuery(hql1);
+
+ String hql2 = "hivedml.hql";
+ String insertSql =
+ String.format(
+ "INSERT INTO %s VALUES "
+ + "(1, 10, 'Hi'), "
+ + "(1, 100, 'Hi Again'), "
+ + "(2, 20, 'Hello'), "
+ + "(3, 30, 'Table'), "
+ + "(4, 40, 'Store');",
+ table);
+ writeSharedFile(hql2, insertSql);
+ executeQuery(hql2);
+
+ checkQueryResults(table, this::executeQuery);
+ }
+
private String executeQuery(String sql) throws Exception {
Container.ExecResult execResult =
getHive()
diff --git
a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
index b73dcfbf5..68f91009f 100644
---
a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
+++
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
@@ -18,12 +18,16 @@
package org.apache.paimon.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import java.util.Map;
import java.util.Properties;
+import static org.apache.hadoop.hive.metastore.Warehouse.getDnsPath;
+
/**
* declaring the name of the key in the parameters of the Hive metastore
table, which indicates
* where the Paimon table is stored.
@@ -51,4 +55,23 @@ public class LocationKeyExtractor {
return propertiesLocation != null ? propertiesLocation : sdLocation;
}
+
+ public static String getLocation(Table table, Configuration conf) throws
MetaException {
+ String sdLocation = table.getSd().getLocation();
+ if (sdLocation != null) {
+ org.apache.hadoop.fs.Path path;
+ path = getDnsPath(new org.apache.hadoop.fs.Path(sdLocation), conf);
+ sdLocation = path.toUri().toString();
+ table.getSd().setLocation(sdLocation);
+ }
+
+ Map<String, String> params = table.getParameters();
+
+ String propertiesLocation = null;
+ if (params != null) {
+ propertiesLocation = params.get(TBPROPERTIES_LOCATION_KEY);
+ }
+
+ return propertiesLocation != null ? propertiesLocation : sdLocation;
+ }
}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
index 2fe4a0f5a..42cccd6a2 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java
@@ -45,6 +45,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Optional;
+import static org.apache.hadoop.hive.metastore.Warehouse.getDnsPath;
import static org.apache.paimon.hive.HiveTypeUtils.typeInfoToLogicalType;
/**
@@ -76,10 +77,12 @@ public class PaimonMetaHook implements HiveMetaHook {
table.getSd().setInputFormat(PaimonInputFormat.class.getCanonicalName());
table.getSd().setOutputFormat(PaimonOutputFormat.class.getCanonicalName());
-
- String location = LocationKeyExtractor.getLocation(table);
+ String location = LocationKeyExtractor.getLocation(table, conf);
if (location == null) {
String warehouse =
conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ org.apache.hadoop.fs.Path hadoopPath =
+ getDnsPath(new org.apache.hadoop.fs.Path(warehouse), conf);
+ warehouse = hadoopPath.toUri().toString();
Identifier identifier = Identifier.create(table.getDbName(),
table.getTableName());
location = AbstractCatalog.dataTableLocation(warehouse,
identifier).toUri().toString();
table.getSd().setLocation(location);
