This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 775357ff3 [hive] Avoid using hive own FileSystem to access the
location of paimon table when using HiveCatalog (#887)
775357ff3 is described below
commit 775357ff32d30973d6d5ff473f161804c6a1d65f
Author: wgcn <[email protected]>
AuthorDate: Thu Apr 27 16:00:35 2023 +0800
[hive] Avoid using hive own FileSystem to access the location of paimon
table when using HiveCatalog (#887)
---
docs/content/how-to/creating-catalogs.md | 7 +-
.../generated/hive_catalog_configuration.html | 9 +
.../org/apache/paimon/catalog/AbstractCatalog.java | 3 +-
.../java/org/apache/paimon/hive/HiveCatalog.java | 57 +++++--
.../org/apache/paimon/hive/HiveCatalogFactory.java | 8 +-
.../org/apache/paimon/hive/HiveCatalogOptions.java | 9 +
.../org/apache/paimon/hive/LocationHelper.java | 38 +++++
.../apache/paimon/hive/StorageLocationHelper.java | 53 ++++++
.../paimon/hive/TBPropertiesLocationHelper.java | 63 +++++++
.../org/apache/paimon/hive/HiveCatalogTest.java | 3 +-
.../apache/paimon/hive/LocationKeyExtractor.java | 28 ++++
.../paimon/hive/HiveCatalogLocationTest.java | 183 +++++++++++++++++++++
.../org/apache/paimon/hive/annotation/Minio.java | 26 +++
.../hive/runner/PaimonEmbeddedHiveRunner.java | 58 ++++++-
paimon-hive/pom.xml | 32 ++++
15 files changed, 550 insertions(+), 27 deletions(-)
diff --git a/docs/content/how-to/creating-catalogs.md
b/docs/content/how-to/creating-catalogs.md
index d24c8ea1c..72855899b 100644
--- a/docs/content/how-to/creating-catalogs.md
+++ b/docs/content/how-to/creating-catalogs.md
@@ -131,4 +131,9 @@ USE paimon.default;
{{< /tab >}}
-{{< /tabs >}}
\ No newline at end of file
+{{< /tabs >}}
+
+If you are using a object storage , and you don't want that the location of
paimon table/database is accessed by the filesystem of hive,
+which may lead to the error such as "No FileSystem for scheme: s3a".
+You can set location in the properties of table/database by the config of
`location-in-properties`. See
+[setting the location of table/database in properties ]({{< ref
"maintenance/configurations#HiveCatalogOptions" >}})
\ No newline at end of file
diff --git a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
index e7ec2f468..f7c3e0cbf 100644
--- a/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
+++ b/docs/layouts/shortcodes/generated/hive_catalog_configuration.html
@@ -20,5 +20,14 @@
<td>String</td>
<td>File directory of the hive-site.xml , used to create
HiveMetastoreClient and security authentication, such as Kerberos, LDAP, Ranger
and so on</td>
</tr>
+ <tr>
+ <td><h5>location-in-properties</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>Setting the location in properties of hive table/database.
+If you don't want to access the location by the filesystem of hive when using
a object storage such as s3,oss
+you can set this option to true.
+</td>
+ </tr>
</tbody>
</table>
diff --git
a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
index 3875f09ff..9e3d25b43 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java
@@ -83,7 +83,8 @@ public abstract class AbstractCatalog implements Catalog {
return FileStoreTableFactory.create(fileIO,
getDataTableLocation(identifier), tableSchema);
}
- protected Path databasePath(String database) {
+ @VisibleForTesting
+ public Path databasePath(String database) {
return new Path(warehouse(), database + DB_SUFFIX);
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index ea77d7e14..c9f42f137 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -18,6 +18,7 @@
package org.apache.paimon.hive;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.AbstractCatalog;
import org.apache.paimon.catalog.CatalogLock;
import org.apache.paimon.catalog.Identifier;
@@ -72,6 +73,7 @@ import java.util.stream.Collectors;
import static org.apache.paimon.hive.HiveCatalogLock.acquireTimeout;
import static org.apache.paimon.hive.HiveCatalogLock.checkMaxSleep;
+import static org.apache.paimon.hive.HiveCatalogOptions.LOCATION_IN_PROPERTIES;
import static org.apache.paimon.options.CatalogOptions.LOCK_ENABLED;
import static org.apache.paimon.options.CatalogOptions.TABLE_TYPE;
import static org.apache.paimon.utils.Preconditions.checkState;
@@ -96,19 +98,36 @@ public class HiveCatalog extends AbstractCatalog {
private final HiveConf hiveConf;
private final String clientClassName;
private final IMetaStoreClient client;
+ private final String warehouse;
- public HiveCatalog(FileIO fileIO, HiveConf hiveConf, String
clientClassName) {
- super(fileIO);
- this.hiveConf = hiveConf;
- this.clientClassName = clientClassName;
- this.client = createClient(hiveConf, clientClassName);
+ private LocationHelper locationHelper;
+
+ public HiveCatalog(FileIO fileIO, HiveConf hiveConf, String
clientClassName, String warehouse) {
+ this(fileIO, hiveConf, clientClassName, Collections.emptyMap(),
warehouse);
}
public HiveCatalog(
- FileIO fileIO, HiveConf hiveConf, String clientClassName,
Map<String, String> options) {
+ FileIO fileIO,
+ HiveConf hiveConf,
+ String clientClassName,
+ Map<String, String> options,
+ String warehouse) {
super(fileIO, options);
this.hiveConf = hiveConf;
this.clientClassName = clientClassName;
+ this.warehouse = warehouse;
+
+ boolean needLocationInProperties =
+ hiveConf.getBoolean(
+ LOCATION_IN_PROPERTIES.key(),
LOCATION_IN_PROPERTIES.defaultValue());
+ if (needLocationInProperties) {
+ locationHelper = new TBPropertiesLocationHelper();
+ } else {
+ // set the warehouse location to the hiveConf
+ hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
warehouse);
+ locationHelper = new StorageLocationHelper();
+ }
+
this.client = createClient(hiveConf, clientClassName);
}
@@ -151,11 +170,13 @@ public class HiveCatalog extends AbstractCatalog {
throws DatabaseAlreadyExistException {
try {
client.createDatabase(convertToDatabase(name));
+
+ locationHelper.createPathIfRequired(databasePath(name), fileIO);
} catch (AlreadyExistsException e) {
if (!ignoreIfExists) {
throw new DatabaseAlreadyExistException(name, e);
}
- } catch (TException e) {
+ } catch (TException | IOException e) {
throw new RuntimeException("Failed to create database " + name, e);
}
}
@@ -167,12 +188,14 @@ public class HiveCatalog extends AbstractCatalog {
if (!cascade && client.getAllTables(name).size() > 0) {
throw new DatabaseNotEmptyException(name);
}
+
+ locationHelper.dropPathIfRequired(databasePath(name), fileIO);
client.dropDatabase(name, true, false, true);
} catch (NoSuchObjectException | UnknownDBException e) {
if (!ignoreIfNotExists) {
throw new DatabaseNotExistException(name, e);
}
- } catch (TException e) {
+ } catch (TException | IOException e) {
throw new RuntimeException("Failed to drop database " + name, e);
}
}
@@ -363,7 +386,7 @@ public class HiveCatalog extends AbstractCatalog {
@Override
protected String warehouse() {
- return hiveConf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ return warehouse;
}
private void checkIdentifierUpperCase(Identifier identifier) {
@@ -410,7 +433,7 @@ public class HiveCatalog extends AbstractCatalog {
private Database convertToDatabase(String name) {
Database database = new Database();
database.setName(name);
- database.setLocationUri(databasePath(name).toString());
+ locationHelper.specifyDatabaseLocation(databasePath(name), database);
return database;
}
@@ -444,19 +467,20 @@ public class HiveCatalog extends AbstractCatalog {
}
private void updateHmsTable(Table table, Identifier identifier,
TableSchema schema) {
- StorageDescriptor sd = convertToStorageDescriptor(identifier, schema);
+ StorageDescriptor sd = convertToStorageDescriptor(schema);
table.setSd(sd);
+
+ // update location
+ locationHelper.specifyTableLocation(table,
getDataTableLocation(identifier).toString());
}
- private StorageDescriptor convertToStorageDescriptor(
- Identifier identifier, TableSchema schema) {
+ private StorageDescriptor convertToStorageDescriptor(TableSchema schema) {
StorageDescriptor sd = new StorageDescriptor();
sd.setCols(
schema.fields().stream()
.map(this::convertToFieldSchema)
.collect(Collectors.toList()));
- sd.setLocation(getDataTableLocation(identifier).toString());
sd.setInputFormat(INPUT_FORMAT_CLASS_NAME);
sd.setOutputFormat(OUTPUT_FORMAT_CLASS_NAME);
@@ -469,6 +493,11 @@ public class HiveCatalog extends AbstractCatalog {
return sd;
}
+ @VisibleForTesting
+ IMetaStoreClient getHmsClient() {
+ return client;
+ }
+
private FieldSchema convertToFieldSchema(DataField dataField) {
return new FieldSchema(
dataField.name(),
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
index e8771df31..2db46caea 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogFactory.java
@@ -69,10 +69,14 @@ public class HiveCatalogFactory implements CatalogFactory {
// always using user-set parameters overwrite hive-site.xml parameters
context.options().toMap().forEach(hiveConf::set);
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
- hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
warehouse.toUri().toString());
String clientClassName = context.options().get(METASTORE_CLIENT_CLASS);
- return new HiveCatalog(fileIO, hiveConf, clientClassName,
context.options().toMap());
+ return new HiveCatalog(
+ fileIO,
+ hiveConf,
+ clientClassName,
+ context.options().toMap(),
+ warehouse.toUri().toString());
}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
index ceebdaa20..431e74dcf 100644
---
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalogOptions.java
@@ -40,5 +40,14 @@ public final class HiveCatalogOptions {
.withDescription(
"File directory of the
core-site.xml、hdfs-site.xml、yarn-site.xml、mapred-site.xml. Currently, only
local file system paths are supported.");
+ public static final ConfigOption<Boolean> LOCATION_IN_PROPERTIES =
+ ConfigOptions.key("location-in-properties")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Setting the location in properties of hive
table/database.\n"
+ + "If you don't want to access the
location by the filesystem of hive when using a object storage such as s3,oss\n"
+ + "you can set this option to true.\n");
+
private HiveCatalogOptions() {}
}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
new file mode 100644
index 000000000..409b79d58
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.io.IOException;
+
+/** helper for specifying the storage location of hive table. */
+public interface LocationHelper {
+ void createPathIfRequired(Path dbPath, FileIO fileIO) throws IOException;
+
+ void dropPathIfRequired(Path path, FileIO fileIO) throws IOException;
+
+ void specifyTableLocation(Table table, String location);
+
+ void specifyDatabaseLocation(Path path, Database database);
+}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
new file mode 100644
index 000000000..6d62d68f3
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.io.IOException;
+
+/** Helper for Setting Location in Hive Table Storage. */
+public final class StorageLocationHelper implements LocationHelper {
+
+ public StorageLocationHelper() {}
+
+ @Override
+ public void createPathIfRequired(Path dbPath, FileIO fileIO) throws
IOException {
+ // do nothing
+ }
+
+ @Override
+ public void dropPathIfRequired(Path path, FileIO fileIO) throws
IOException {
+ // do nothing
+ }
+
+ @Override
+ public void specifyTableLocation(Table table, String location) {
+ table.getSd().setLocation(location);
+ }
+
+ @Override
+ public void specifyDatabaseLocation(Path path, Database database) {
+ database.setLocationUri(path.toString());
+ }
+}
diff --git
a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
new file mode 100644
index 000000000..a374907ed
--- /dev/null
+++
b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/** Helper for Setting Location in Hive Table Properties. */
+public final class TBPropertiesLocationHelper implements LocationHelper {
+
+ public TBPropertiesLocationHelper() {}
+
+ @Override
+ public void createPathIfRequired(Path path, FileIO fileIO) throws
IOException {
+ if (!fileIO.exists(path)) {
+ fileIO.mkdirs(path);
+ }
+ }
+
+ @Override
+ public void dropPathIfRequired(Path path, FileIO fileIO) throws
IOException {
+ if (fileIO.exists(path)) {
+ fileIO.delete(path, true);
+ }
+ }
+
+ @Override
+ public void specifyTableLocation(Table table, String location) {
+
table.getParameters().put(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY,
location);
+ }
+
+ @Override
+ public void specifyDatabaseLocation(Path path, Database database) {
+ HashMap<String, String> properties = new HashMap<>();
+ if (database.getParameters() != null) {
+ properties.putAll(database.getParameters());
+ }
+ properties.put(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY,
path.toString());
+ database.setParameters(properties);
+ }
+}
diff --git
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index 6695e1d3e..b56ca222a 100644
---
a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++
b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -44,14 +44,13 @@ public class HiveCatalogTest extends CatalogTestBase {
HiveConf hiveConf = new HiveConf();
String jdoConnectionURL = "jdbc:derby:memory:" + UUID.randomUUID();
hiveConf.setVar(METASTORECONNECTURLKEY, jdoConnectionURL +
";create=true");
- hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouse);
HiveMetaStoreClient metaStoreClient = new
HiveMetaStoreClient(hiveConf);
String metastoreClientClass =
"org.apache.hadoop.hive.metastore.HiveMetaStoreClient";
try (MockedStatic<HiveCatalog> mocked =
Mockito.mockStatic(HiveCatalog.class)) {
mocked.when(() -> HiveCatalog.createClient(hiveConf,
metastoreClientClass))
.thenReturn(metaStoreClient);
}
- catalog = new HiveCatalog(fileIO, hiveConf, metastoreClientClass);
+ catalog = new HiveCatalog(fileIO, hiveConf, metastoreClientClass,
warehouse);
}
@Test
diff --git
a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
new file mode 100644
index 000000000..a5663a18d
--- /dev/null
+++
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/LocationKeyExtractor.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+/**
+ * declaring the name of the key in the parameters of the Hive metastore
table, which indicates
+ * where the Paimon table is stored.
+ */
+public class LocationKeyExtractor {
+ // special at the tbproperties with the name paimon_meta_location.
+ public static final String TBPROPERTIES_LOCATION_KEY =
"paimon_meta_location";
+}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogLocationTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogLocationTest.java
new file mode 100644
index 000000000..138ca40bb
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogLocationTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.hive.annotation.Minio;
+import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.s3.MinioTestContainer;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.apache.paimon.shade.guava30.com.google.common.collect.Sets;
+
+import com.klarna.hiverunner.HiveShell;
+import com.klarna.hiverunner.annotations.HiveSQL;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/** Test for HiveCatalog location. */
+@RunWith(PaimonEmbeddedHiveRunner.class)
+public class HiveCatalogLocationTest {
+ @HiveSQL(files = {})
+ private static HiveShell hiveShell;
+
+ @Minio private static MinioTestContainer minioTestContainer;
+
+ public static final String HIVE_CONF = "/hive-conf";
+
+ private HiveCatalog catalog;
+
+ private IMetaStoreClient hmsClient;
+
+ private String path;
+
+ private FileIO fileIO;
+
+ @Before
+ public void before() throws IOException {
+ path = minioTestContainer.getS3UriForDefaultBucket() + "/" +
UUID.randomUUID();
+
+ Options conf = new Options();
+ conf.set(CatalogOptions.WAREHOUSE, path);
+ conf.set(CatalogOptions.METASTORE, "hive");
+ conf.set(CatalogOptions.URI, "");
+ conf.set(
+ HiveCatalogOptions.HIVE_CONF_DIR,
+ hiveShell.getBaseDir().getRoot().getPath() + HIVE_CONF);
+ conf.set(HiveCatalogOptions.LOCATION_IN_PROPERTIES, true);
+
+ for (Map.Entry<String, String> stringStringEntry :
+ minioTestContainer.getS3ConfigOptions().entrySet()) {
+ conf.set(stringStringEntry.getKey(), stringStringEntry.getValue());
+ }
+
+ // create CatalogContext using the options
+ CatalogContext catalogContext = CatalogContext.create(conf);
+
+ Path warehouse = new Path(path);
+ fileIO = getFileIO(catalogContext, warehouse);
+ fileIO.mkdirs(warehouse);
+
+ HiveCatalogFactory hiveCatalogFactory = new HiveCatalogFactory();
+ catalog = (HiveCatalog) hiveCatalogFactory.create(fileIO, warehouse,
catalogContext);
+ hmsClient = catalog.getHmsClient();
+ }
+
+ private static FileIO getFileIO(CatalogContext catalogContext, Path
warehouse) {
+ FileIO fileIO;
+ try {
+ fileIO = FileIO.get(warehouse, catalogContext);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ return fileIO;
+ }
+
+ @After
+ public void after() throws Exception {
+ catalog.close();
+ }
+
+ @Test
+ public void testDBLocation() throws Exception {
+
+ Set<String> dbs = Sets.newHashSet("db1", "db2", "db3", "db4", "db5");
+ List<Path> paths = new ArrayList<>();
+ for (String db : dbs) {
+ catalog.createDatabase(db, true);
+ Assert.assertNotNull(hmsClient.getDatabase(db));
+
+ Path actual = catalog.databasePath(db);
+ Path expected = new Path(this.path + "/" + db + ".db");
+ Assert.assertTrue(fileIO.exists(expected));
+ Assert.assertEquals(expected, actual);
+
+ paths.add(expected);
+ }
+
+ HashSet<String> dbsExpected = Sets.newHashSet("db1", "db2", "db3",
"db4", "db5", "default");
+ Assert.assertEquals(Sets.newHashSet(catalog.listDatabases()),
dbsExpected);
+
+ for (String db : dbs) {
+ catalog.dropDatabase(db, false, true);
+ }
+
+ for (Path p : paths) {
+ Assert.assertFalse(fileIO.exists(p));
+ }
+
+ Assert.assertEquals(Sets.newHashSet(catalog.listDatabases()),
Sets.newHashSet("default"));
+ }
+
+ @Test
+ public void testTableLocation() throws Exception {
+
+ String db = "db";
+ String table = "table";
+
+ catalog.createDatabase(db, true);
+
+ RowType rowType = RowType.of(new DataType[] {DataTypes.INT()}, new
String[] {"aaa"});
+ Identifier tableIdentifier = Identifier.create(db, table);
+
+ // create table
+ catalog.createTable(
+ tableIdentifier,
+ new Schema(
+ rowType.getFields(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ new HashMap<>(),
+ ""),
+ false);
+
+ Table hmsClientTablea =
+ hmsClient.getTable(
+ tableIdentifier.getDatabaseName(),
tableIdentifier.getObjectName());
+ String location =
+
hmsClientTablea.getParameters().get(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY);
+ String expected = this.path + "/" + db + ".db" + "/" + table;
+ Assert.assertTrue(fileIO.exists(new Path(expected)));
+ Assert.assertEquals(expected, location);
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/annotation/Minio.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/annotation/Minio.java
new file mode 100644
index 000000000..88c6eebc3
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/annotation/Minio.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.annotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+/** Annotation for Minio Container. */
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Minio {}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
index 3d161cac9..fbb29cb3c 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
@@ -18,6 +18,8 @@
package org.apache.paimon.hive.runner;
+import org.apache.paimon.hive.annotation.Minio;
+import org.apache.paimon.s3.MinioTestContainer;
import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.shade.guava30.com.google.common.io.Resources;
@@ -72,9 +74,7 @@ import static org.reflections.ReflectionUtils.withAnnotation;
public class PaimonEmbeddedHiveRunner extends BlockJUnit4ClassRunner {
private static final Logger LOGGER =
LoggerFactory.getLogger(PaimonEmbeddedHiveRunner.class);
- private HiveShellContainer container;
private final HiveRunnerConfig config = new HiveRunnerConfig();
- protected HiveServerContext context;
public PaimonEmbeddedHiveRunner(Class<?> clazz) throws InitializationError
{
super(clazz);
@@ -85,21 +85,45 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
// need to load hive runner config before the context is inited
loadAnnotatesHiveRunnerConfig(getTestClass().getJavaClass());
final TemporaryFolder temporaryFolder = new TemporaryFolder();
- context = new PaimonEmbeddedHiveServerContext(temporaryFolder, config);
List<TestRule> rules = super.classRules();
ExternalResource hiveShell =
new ExternalResource() {
+ private HiveShellContainer innerContainer;
+
@Override
protected void before() throws Throwable {
- container =
-
createHiveServerContainer(getTestClass().getJavaClass(), context);
+ PaimonEmbeddedHiveServerContext
paimonEmbeddedHiveServerContext =
+ new
PaimonEmbeddedHiveServerContext(temporaryFolder, config);
+ innerContainer =
+ createHiveServerContainer(
+ getTestClass().getJavaClass(),
+ paimonEmbeddedHiveServerContext);
+ }
+
+ @Override
+ protected void after() {
+ tearDownHiveContainer(innerContainer);
+ }
+ };
+
+ ExternalResource minio =
+ new ExternalResource() {
+ public MinioTestContainer minioTestContainer;
+
+ @Override
+ protected void before() {
+ minioTestContainer =
setMinioTestContainer(getTestClass().getJavaClass());
}
@Override
protected void after() {
- tearDown();
+ if (minioTestContainer != null &&
minioTestContainer.isRunning()) {
+ minioTestContainer.close();
+ }
}
};
+
+ rules.add(minio);
rules.add(hiveShell);
rules.add(temporaryFolder);
return rules;
@@ -134,7 +158,7 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
}
}
- private void tearDown() {
+ private void tearDownHiveContainer(HiveShellContainer container) {
if (container != null) {
LOGGER.info("Tearing down {}", getName());
try {
@@ -342,6 +366,26 @@ public class PaimonEmbeddedHiveRunner extends
BlockJUnit4ClassRunner {
}
}
+ private MinioTestContainer setMinioTestContainer(final Class testClass) {
+ Set<Field> allFields = ReflectionUtils.getAllFields(testClass,
withAnnotation(Minio.class));
+
+ Preconditions.checkState(
+ allFields.size() <= 1,
+ "At most one field of type MinioTestContainer should to be
annotated with @MinIO");
+ MinioTestContainer minioTestContainer = null;
+ if (!allFields.isEmpty()) {
+ minioTestContainer = new MinioTestContainer();
+ minioTestContainer.start();
+
+ Field field = allFields.iterator().next();
+ Preconditions.checkState(
+ ReflectionUtils.isOfType(field, MinioTestContainer.class),
+ "Field annotated with @MinIO should be of type
MinioTestContainer");
+ ReflectionUtils.setStaticField(testClass, field.getName(),
minioTestContainer);
+ }
+ return minioTestContainer;
+ }
+
/**
* Used as a handle for the HiveShell field in the test case so that we
may set it once the
* HiveShell has been instantiated.
diff --git a/paimon-hive/pom.xml b/paimon-hive/pom.xml
index fc888bbb2..b0fc8bc17 100644
--- a/paimon-hive/pom.xml
+++ b/paimon-hive/pom.xml
@@ -48,6 +48,38 @@ under the License.
<hive.version>2.3.9</hive.version>
<hiverunner.version>4.0.0</hiverunner.version>
<reflections.version>0.9.8</reflections.version>
+ <aws.version>1.12.319</aws.version>
</properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-s3</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-s3</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <version>${aws.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
</project>
