This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f17193077 [hive] Get rid of flink-connector-hive dependency in
paimon-hive (#776)
f17193077 is described below
commit f17193077bd6be612ce413f524de92d526f457f8
Author: Kerwin <[email protected]>
AuthorDate: Mon Apr 3 14:10:17 2023 +0800
[hive] Get rid of flink-connector-hive dependency in paimon-hive (#776)
---
paimon-hive/paimon-hive-connector-2.3/pom.xml | 16 +-
.../apache/paimon/hive/Hive23CatalogITCase.java | 4 +-
paimon-hive/paimon-hive-connector-3.1/pom.xml | 16 +-
.../apache/paimon/hive/Hive31CatalogITCase.java | 4 +-
paimon-hive/paimon-hive-connector-common/pom.xml | 17 +-
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 4 +-
.../paimon/hive/PaimonStorageHandlerITCase.java | 4 +-
.../hive/runner/PaimonEmbeddedHiveRunner.java | 353 +++++++++++++++++++++
.../runner/PaimonEmbeddedHiveServerContext.java | 233 ++++++++++++++
.../src/test/resources/hive-site.xml | 53 ++++
10 files changed, 656 insertions(+), 48 deletions(-)
diff --git a/paimon-hive/paimon-hive-connector-2.3/pom.xml
b/paimon-hive/paimon-hive-connector-2.3/pom.xml
index 75e950093..c33297b37 100644
--- a/paimon-hive/paimon-hive-connector-2.3/pom.xml
+++ b/paimon-hive/paimon-hive-connector-2.3/pom.xml
@@ -74,31 +74,23 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <artifactId>flink-connector-files</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-table-runtime</artifactId>
- <version>${test.flink.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+ <artifactId>flink-table-runtime</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
@@ -480,7 +472,7 @@ under the License.
</dependency>
<!--
- hive-exec must stay after flink-connector-hive and avro to avoid
conflicts
+ hive-exec must stay after avro to avoid conflicts
-->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
index 2df418e37..b2d38da4b 100644
---
a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
@@ -19,11 +19,11 @@
package org.apache.paimon.hive;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.schema.SchemaManager;
import com.klarna.hiverunner.annotations.HiveRunnerSetup;
import com.klarna.hiverunner.config.HiveRunnerConfig;
-import org.apache.flink.connectors.hive.FlinkEmbeddedHiveRunner;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.internal.TableEnvironmentImpl;
@@ -42,7 +42,7 @@ import static
org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** IT cases for using Paimon {@link HiveCatalog} together with Paimon Hive
2.3 connector. */
-@RunWith(FlinkEmbeddedHiveRunner.class)
+@RunWith(PaimonEmbeddedHiveRunner.class)
public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
@HiveRunnerSetup
diff --git a/paimon-hive/paimon-hive-connector-3.1/pom.xml
b/paimon-hive/paimon-hive-connector-3.1/pom.xml
index 942cc340f..4ff741e84 100644
--- a/paimon-hive/paimon-hive-connector-3.1/pom.xml
+++ b/paimon-hive/paimon-hive-connector-3.1/pom.xml
@@ -81,31 +81,23 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <artifactId>flink-connector-files</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-table-runtime</artifactId>
- <version>${test.flink.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+ <artifactId>flink-table-runtime</artifactId>
<version>${test.flink.version}</version>
<scope>test</scope>
- <type>test-jar</type>
</dependency>
<dependency>
@@ -498,7 +490,7 @@ under the License.
</dependency>
<!--
- hive-exec must stay after flink-connector-hive and avro to avoid
conflicts
+ hive-exec must stay after avro to avoid conflicts
-->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
index 26d24c74b..84860dcde 100644
---
a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
+++
b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
@@ -19,11 +19,11 @@
package org.apache.paimon.hive;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.schema.SchemaManager;
import com.klarna.hiverunner.annotations.HiveRunnerSetup;
import com.klarna.hiverunner.config.HiveRunnerConfig;
-import org.apache.flink.connectors.hive.FlinkEmbeddedHiveRunner;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.internal.TableEnvironmentImpl;
@@ -42,7 +42,7 @@ import static
org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** IT cases for using Paimon {@link HiveCatalog} together with Paimon Hive
3.1 connector. */
-@RunWith(FlinkEmbeddedHiveRunner.class)
+@RunWith(PaimonEmbeddedHiveRunner.class)
public class Hive31CatalogITCase extends HiveCatalogITCaseBase {
@HiveRunnerSetup
diff --git a/paimon-hive/paimon-hive-connector-common/pom.xml
b/paimon-hive/paimon-hive-connector-common/pom.xml
index 5736a6000..d9fa60865 100644
--- a/paimon-hive/paimon-hive-connector-common/pom.xml
+++ b/paimon-hive/paimon-hive-connector-common/pom.xml
@@ -121,21 +121,6 @@ under the License.
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
- <version>${test.flink.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
-
<artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
- <version>${test.flink.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-orc</artifactId>
@@ -477,7 +462,7 @@ under the License.
</dependency>
<!--
- hive-exec must stay after flink-connector-hive and avro to avoid
conflicts
+ hive-exec must stay after avro to avoid conflicts
-->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index 86b7d4995..d50bb16e2 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -24,10 +24,10 @@ import org.apache.paimon.catalog.CatalogLock;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.flink.FlinkCatalog;
import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.annotations.HiveSQL;
-import org.apache.flink.connectors.hive.FlinkEmbeddedHiveRunner;
import org.apache.flink.core.fs.Path;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;
@@ -56,7 +56,7 @@ import static
org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
/** IT cases for using Paimon {@link HiveCatalog} together with Paimon Hive
connector. */
-@RunWith(FlinkEmbeddedHiveRunner.class)
+@RunWith(PaimonEmbeddedHiveRunner.class)
public abstract class HiveCatalogITCaseBase {
@Rule public TemporaryFolder folder = new TemporaryFolder();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
index 28f519b1b..760388d31 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
@@ -26,6 +26,7 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.hive.mapred.PaimonInputFormat;
import org.apache.paimon.hive.objectinspector.PaimonObjectInspectorFactory;
+import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.Table;
@@ -39,7 +40,6 @@ import org.apache.paimon.types.RowType;
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.annotations.HiveSQL;
-import org.apache.flink.connectors.hive.FlinkEmbeddedHiveRunner;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
@@ -65,7 +65,7 @@ import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
/** IT cases for {@link PaimonStorageHandler} and {@link PaimonInputFormat}. */
-@RunWith(FlinkEmbeddedHiveRunner.class)
+@RunWith(PaimonEmbeddedHiveRunner.class)
public class PaimonStorageHandlerITCase {
@ClassRule public static TemporaryFolder folder = new TemporaryFolder();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
new file mode 100644
index 000000000..3289c6a26
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.runner;
+
+import org.apache.paimon.utils.Preconditions;
+
+import com.klarna.hiverunner.HiveServerContainer;
+import com.klarna.hiverunner.HiveServerContext;
+import com.klarna.hiverunner.HiveShell;
+import com.klarna.hiverunner.HiveShellContainer;
+import com.klarna.hiverunner.annotations.HiveProperties;
+import com.klarna.hiverunner.annotations.HiveResource;
+import com.klarna.hiverunner.annotations.HiveRunnerSetup;
+import com.klarna.hiverunner.annotations.HiveSQL;
+import com.klarna.hiverunner.annotations.HiveSetupScript;
+import com.klarna.hiverunner.builder.HiveShellBuilder;
+import com.klarna.hiverunner.config.HiveRunnerConfig;
+import com.klarna.reflection.ReflectionUtils;
+import org.apache.flink.shaded.guava30.com.google.common.io.Resources;
+import org.junit.Ignore;
+import org.junit.internal.AssumptionViolatedException;
+import org.junit.internal.runners.model.EachTestNotifier;
+import org.junit.rules.ExternalResource;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.reflections.ReflectionUtils.withAnnotation;
+
+/**
+ * JUnit 4 runner that runs hive sql on a HiveServer residing in this JVM. No
external dependencies
+ * needed. Inspired by StandaloneHiveRunner.java (almost copied).
+ */
+public class PaimonEmbeddedHiveRunner extends BlockJUnit4ClassRunner {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(PaimonEmbeddedHiveRunner.class);
+ private HiveShellContainer container;
+ private final HiveRunnerConfig config = new HiveRunnerConfig();
+ protected HiveServerContext context;
+
+ public PaimonEmbeddedHiveRunner(Class<?> clazz) throws InitializationError
{
+ super(clazz);
+ }
+
+ @Override
+ protected List<TestRule> classRules() {
+ // need to load hive runner config before the context is inited
+ loadAnnotatesHiveRunnerConfig(getTestClass().getJavaClass());
+ final TemporaryFolder temporaryFolder = new TemporaryFolder();
+ context = new PaimonEmbeddedHiveServerContext(temporaryFolder, config);
+ List<TestRule> rules = super.classRules();
+ ExternalResource hiveShell =
+ new ExternalResource() {
+ @Override
+ protected void before() throws Throwable {
+ container =
+
createHiveServerContainer(getTestClass().getJavaClass(), context);
+ }
+
+ @Override
+ protected void after() {
+ tearDown();
+ }
+ };
+ rules.add(hiveShell);
+ rules.add(temporaryFolder);
+ return rules;
+ }
+
+ @Override
+ protected void runChild(final FrameworkMethod method, RunNotifier
notifier) {
+ Description description = describeChild(method);
+ if (method.getAnnotation(Ignore.class) != null) {
+ notifier.fireTestIgnored(description);
+ } else {
+ EachTestNotifier eachNotifier = new EachTestNotifier(notifier,
description);
+ eachNotifier.fireTestStarted();
+ try {
+ runTestMethod(method, eachNotifier);
+ } finally {
+ eachNotifier.fireTestFinished();
+ }
+ }
+ }
+
+ /** Runs a {@link Statement} that represents a leaf (aka atomic) test. */
+ private void runTestMethod(FrameworkMethod method, EachTestNotifier
notifier) {
+ Statement statement = methodBlock(method);
+
+ try {
+ statement.evaluate();
+ } catch (AssumptionViolatedException e) {
+ notifier.addFailedAssumption(e);
+ } catch (Throwable e) {
+ notifier.addFailure(e);
+ }
+ }
+
+ private void tearDown() {
+ if (container != null) {
+ LOGGER.info("Tearing down {}", getName());
+ try {
+ container.tearDown();
+ } catch (Throwable e) {
+ LOGGER.warn("Tear down failed: " + e.getMessage(), e);
+ }
+ }
+ }
+
+ /**
+ * Traverses the test class annotations. Will inject a HiveShell in the
test case that envelopes
+ * the HiveServer.
+ */
+ private HiveShellContainer createHiveServerContainer(
+ final Class testClass, HiveServerContext context) throws Exception
{
+
+ context.init();
+
+ final HiveServerContainer hiveServerContainer = new
HiveServerContainer(context);
+
+ HiveShellBuilder hiveShellBuilder = new HiveShellBuilder();
+ setCommandShellEmulation(hiveShellBuilder, config);
+
+ HiveShellField shellSetter = loadScriptsUnderTest(testClass,
hiveShellBuilder);
+
+ hiveShellBuilder.setHiveServerContainer(hiveServerContainer);
+
+ loadAnnotatedResources(testClass, hiveShellBuilder);
+
+ loadAnnotatedProperties(testClass, hiveShellBuilder);
+
+ loadAnnotatedSetupScripts(testClass, hiveShellBuilder);
+
+ // Build shell
+ final HiveShellContainer shell = hiveShellBuilder.buildShell();
+
+ // Set shell
+ shellSetter.setShell(shell);
+
+ if (shellSetter.isAutoStart()) {
+ shell.start();
+ }
+
+ return shell;
+ }
+
+ private void setCommandShellEmulation(HiveShellBuilder builder,
HiveRunnerConfig config)
+ throws Exception {
+ Method method =
config.getClass().getDeclaredMethod("getCommandShellEmulator");
+ Object emulator = method.invoke(config);
+ Class emulatorClz =
Class.forName("com.klarna.hiverunner.sql.cli.CommandShellEmulator");
+ method =
builder.getClass().getDeclaredMethod("setCommandShellEmulation", emulatorClz);
+ method.invoke(builder, emulator);
+ }
+
+ private void loadAnnotatesHiveRunnerConfig(Class testClass) {
+ Set<Field> fields =
+ ReflectionUtils.getAllFields(testClass,
withAnnotation(HiveRunnerSetup.class));
+ Preconditions.checkState(
+ fields.size() <= 1,
+ "Exact one field of type HiveRunnerConfig should to be
annotated with @HiveRunnerSetup");
+
+ // Override the config with test case config. Taking care to not
replace the config instance
+ // since it
+ // has been passes around and referenced by some of the other test
rules.
+ if (!fields.isEmpty()) {
+ Field field = fields.iterator().next();
+ Preconditions.checkState(
+ ReflectionUtils.isOfType(field, HiveRunnerConfig.class),
+ "Field annotated with @HiveRunnerSetup should be of type
HiveRunnerConfig");
+ config.override(
+ ReflectionUtils.getStaticFieldValue(
+ testClass, field.getName(),
HiveRunnerConfig.class));
+ }
+ }
+
+ private HiveShellField loadScriptsUnderTest(
+ final Class testClass, HiveShellBuilder hiveShellBuilder) {
+ try {
+ Set<Field> fields =
+ ReflectionUtils.getAllFields(testClass,
withAnnotation(HiveSQL.class));
+
+ Preconditions.checkState(
+ fields.size() == 1, "Exactly one field should to be
annotated with @HiveSQL");
+
+ final Field field = fields.iterator().next();
+ List<Path> scripts = new ArrayList<>();
+ HiveSQL annotation = field.getAnnotation(HiveSQL.class);
+ for (String scriptFilePath : annotation.files()) {
+ Path file =
Paths.get(Resources.getResource(scriptFilePath).toURI());
+ Preconditions.checkState(Files.exists(file), "File " + file +
" does not exist");
+ scripts.add(file);
+ }
+
+ Charset charset =
+ annotation.encoding().equals("")
+ ? Charset.defaultCharset()
+ : Charset.forName(annotation.encoding());
+
+ final boolean isAutoStart = annotation.autoStart();
+
+ hiveShellBuilder.setScriptsUnderTest(scripts, charset);
+
+ return new HiveShellField() {
+ @Override
+ public void setShell(HiveShell shell) {
+ ReflectionUtils.setStaticField(testClass, field.getName(),
shell);
+ }
+
+ @Override
+ public boolean isAutoStart() {
+ return isAutoStart;
+ }
+ };
+ } catch (Throwable t) {
+ throw new IllegalArgumentException(
+ "Failed to init field annotated with @HiveSQL: " +
t.getMessage(), t);
+ }
+ }
+
+ private void loadAnnotatedSetupScripts(Class testClass, HiveShellBuilder
hiveShellBuilder) {
+ Set<Field> setupScriptFields =
+ ReflectionUtils.getAllFields(testClass,
withAnnotation(HiveSetupScript.class));
+ for (Field setupScriptField : setupScriptFields) {
+ if (ReflectionUtils.isOfType(setupScriptField, String.class)) {
+ String script =
+ ReflectionUtils.getStaticFieldValue(
+ testClass, setupScriptField.getName(),
String.class);
+ hiveShellBuilder.addSetupScript(script);
+ } else if (ReflectionUtils.isOfType(setupScriptField, File.class)
+ || ReflectionUtils.isOfType(setupScriptField, Path.class))
{
+ Path path = getMandatoryPathFromField(testClass,
setupScriptField);
+ hiveShellBuilder.addSetupScript(readAll(path));
+ } else {
+ throw new IllegalArgumentException(
+ "Field annotated with @HiveSetupScript currently only
supports type String, File and Path");
+ }
+ }
+ }
+
+ private static String readAll(Path path) {
+ try {
+ return new String(Files.readAllBytes(path),
StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to read " + path + ": " +
e.getMessage(), e);
+ }
+ }
+
+ private void loadAnnotatedResources(Class testClass, HiveShellBuilder
workFlowBuilder)
+ throws IOException {
+ Set<Field> fields =
+ ReflectionUtils.getAllFields(testClass,
withAnnotation(HiveResource.class));
+
+ for (Field resourceField : fields) {
+
+ HiveResource annotation =
resourceField.getAnnotation(HiveResource.class);
+ String targetFile = annotation.targetFile();
+
+ if (ReflectionUtils.isOfType(resourceField, String.class)) {
+ String data =
+ ReflectionUtils.getStaticFieldValue(
+ testClass, resourceField.getName(),
String.class);
+ workFlowBuilder.addResource(targetFile, data);
+ } else if (ReflectionUtils.isOfType(resourceField, File.class)
+ || ReflectionUtils.isOfType(resourceField, Path.class)) {
+ Path dataFile = getMandatoryPathFromField(testClass,
resourceField);
+ workFlowBuilder.addResource(targetFile, dataFile);
+ } else {
+ throw new IllegalArgumentException(
+ "Fields annotated with @HiveResource currently only
supports field type String, File or Path");
+ }
+ }
+ }
+
+ private Path getMandatoryPathFromField(Class testClass, Field
resourceField) {
+ Path path;
+ if (ReflectionUtils.isOfType(resourceField, File.class)) {
+ File dataFile =
+ ReflectionUtils.getStaticFieldValue(
+ testClass, resourceField.getName(), File.class);
+ path = Paths.get(dataFile.toURI());
+ } else if (ReflectionUtils.isOfType(resourceField, Path.class)) {
+ path =
+ ReflectionUtils.getStaticFieldValue(
+ testClass, resourceField.getName(), Path.class);
+ } else {
+ throw new IllegalArgumentException(
+ "Only Path or File type is allowed on annotated field " +
resourceField);
+ }
+
+ Preconditions.checkArgument(Files.exists(path), "File %s does not
exist", path);
+ return path;
+ }
+
+ private void loadAnnotatedProperties(Class testClass, HiveShellBuilder
workFlowBuilder) {
+ for (Field hivePropertyField :
+ ReflectionUtils.getAllFields(testClass,
withAnnotation(HiveProperties.class))) {
+ Preconditions.checkState(
+ ReflectionUtils.isOfType(hivePropertyField, Map.class),
+ "Field annotated with @HiveProperties should be of type
Map<String, String>");
+ workFlowBuilder.putAllProperties(
+ ReflectionUtils.getStaticFieldValue(
+ testClass, hivePropertyField.getName(),
Map.class));
+ }
+ }
+
+ /**
+ * Used as a handle for the HiveShell field in the test case so that we
may set it once the
+ * HiveShell has been instantiated.
+ */
+ interface HiveShellField {
+ void setShell(HiveShell shell);
+
+ boolean isAutoStart();
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
new file mode 100644
index 000000000..3de27f66e
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveServerContext.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.runner;
+
+import com.klarna.hiverunner.HiveServerContext;
+import com.klarna.hiverunner.config.HiveRunnerConfig;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HADOOPBIN;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVECONVERTJOIN;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEHISTORYFILELOC;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEMETADATAONLYQUERIES;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEOPTINDEXFILTER;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESKEWJOIN;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSAUTOGATHER;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_CBO_ENABLED;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_INFER_BUCKET_SORT;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.LOCALSCRATCHDIR;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORECONNECTURLKEY;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREWAREHOUSE;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_VALIDATE_COLUMNS;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_VALIDATE_CONSTRAINTS;
+import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTORE_VALIDATE_TABLES;
+import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.SCRATCHDIR;
+
+/** HiveServerContext used by PaimonEmbeddedHiveRunner. */
+public class PaimonEmbeddedHiveServerContext implements HiveServerContext {
+
+ private final HiveConf hiveConf = new HiveConf();
+
+ private final TemporaryFolder basedir;
+ private final HiveRunnerConfig hiveRunnerConfig;
+ private boolean inited = false;
+
+ PaimonEmbeddedHiveServerContext(TemporaryFolder basedir, HiveRunnerConfig
hiveRunnerConfig) {
+ this.basedir = basedir;
+ this.hiveRunnerConfig = hiveRunnerConfig;
+ }
+
+ @Override
+ public final void init() {
+ if (!inited) {
+
+ configureMiscHiveSettings();
+
+ configureMetaStore();
+
+ configureMrExecutionEngine();
+
+ configureJavaSecurityRealm();
+
+ configureSupportConcurrency();
+
+ configureFileSystem();
+
+ configureAssertionStatus();
+
+ overrideHiveConf();
+
+ setHiveSitePath();
+ }
+ inited = true;
+ }
+
+ // Some Hive code may create HiveConf instances relying on the hive-site
in classpath. Make sure
+ // such code can
+ // read the configurations we set here.
+ private void setHiveSitePath() {
+ File hiveSite = new File(newFolder(basedir, "hive-conf"),
"hive-site.xml");
+ try (FileOutputStream outputStream = new FileOutputStream(hiveSite)) {
+ hiveConf.writeXml(outputStream);
+ HiveConf.setHiveSiteLocation(hiveSite.toURI().toURL());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to write hive-site.xml", e);
+ }
+ }
+
+ private void configureMiscHiveSettings() {
+ hiveConf.setBoolVar(HIVESTATSAUTOGATHER, false);
+
+ // Turn off CBO so we don't depend on calcite
+ hiveConf.setBoolVar(HIVE_CBO_ENABLED, false);
+
+ // Disable to get rid of clean up exception when stopping the Session.
+ hiveConf.setBoolVar(HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false);
+
+ hiveConf.setVar(HADOOPBIN, "NO_BIN!");
+
+ // To avoid https://issues.apache.org/jira/browse/HIVE-13185 when
loading data into tables
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVECHECKFILEFORMAT, false);
+ }
+
+ private void overrideHiveConf() {
+ for (Map.Entry<String, String> hiveConfEntry :
+ hiveRunnerConfig.getHiveConfSystemOverride().entrySet()) {
+ hiveConf.set(hiveConfEntry.getKey(), hiveConfEntry.getValue());
+ }
+ }
+
+ private void configureMrExecutionEngine() {
+
+ /*
+ * Switch off all optimizers otherwise we didn't
+ * manage to contain the map reduction within this JVM.
+ */
+ hiveConf.setBoolVar(HIVE_INFER_BUCKET_SORT, false);
+ hiveConf.setBoolVar(HIVEMETADATAONLYQUERIES, false);
+ hiveConf.setBoolVar(HIVEOPTINDEXFILTER, false);
+ hiveConf.setBoolVar(HIVECONVERTJOIN, false);
+ hiveConf.setBoolVar(HIVESKEWJOIN, false);
+
+ // Defaults to a 1000 millis sleep in. We can speed up the tests a bit
by setting this to 1
+ // millis instead.
+ // org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper.
+ hiveConf.setLongVar(HiveConf.ConfVars.HIVECOUNTERSPULLINTERVAL, 1L);
+
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_RPC_QUERY_PLAN, true);
+ }
+
+ private void configureJavaSecurityRealm() {
+ // These three properties gets rid of: 'Unable to load realm info from
SCDynamicStore'
+ // which seems to have a timeout of about 5 secs.
+ System.setProperty("java.security.krb5.realm", "EXAMPLE.COM");
+ System.setProperty("java.security.krb5.kdc", "kdc");
+ System.setProperty("java.security.krb5.conf", "/dev/null");
+ }
+
+ private void configureAssertionStatus() {
+ ClassLoader.getSystemClassLoader()
+
.setPackageAssertionStatus("org.apache.hadoop.hive.serde2.objectinspector",
false);
+ }
+
+ private void configureSupportConcurrency() {
+ hiveConf.setBoolVar(HIVE_SUPPORT_CONCURRENCY, false);
+ }
+
+ private void configureMetaStore() {
+
+ String jdbcDriver =
org.apache.derby.jdbc.EmbeddedDriver.class.getName();
+ try {
+ Class.forName(jdbcDriver);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ // set JDO configs
+ String jdoConnectionURL = "jdbc:derby:memory:" +
UUID.randomUUID().toString();
+ hiveConf.setVar(METASTORECONNECTURLKEY, jdoConnectionURL +
";create=true");
+
+ hiveConf.setBoolVar(METASTORE_VALIDATE_CONSTRAINTS, true);
+ hiveConf.setBoolVar(METASTORE_VALIDATE_COLUMNS, true);
+ hiveConf.setBoolVar(METASTORE_VALIDATE_TABLES, true);
+
+ // disable authorization to avoid NPE
+ hiveConf.set(
+ HIVE_AUTHORIZATION_MANAGER.varname,
+
"org.apache.hive.hcatalog.storagehandler.DummyHCatAuthProvider");
+
+ // disable notification event poll
+ hiveConf.set("hive.notification.event.poll.interval", "0s");
+ }
+
+ private void configureFileSystem() {
+
+ createAndSetFolderProperty(METASTOREWAREHOUSE, "warehouse", hiveConf,
basedir);
+ createAndSetFolderProperty(SCRATCHDIR, "scratchdir", hiveConf,
basedir);
+ createAndSetFolderProperty(LOCALSCRATCHDIR, "localscratchdir",
hiveConf, basedir);
+ createAndSetFolderProperty(HIVEHISTORYFILELOC, "tmp", hiveConf,
basedir);
+
+ // HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS is removed from Hive 3.1.0
+ hiveConf.setBoolean("hive.warehouse.subdir.inherit.perms", true);
+
+ createAndSetFolderProperty("hadoop.tmp.dir", "hadooptmp", hiveConf,
basedir);
+ createAndSetFolderProperty("test.log.dir", "logs", hiveConf, basedir);
+ }
+
+ private File newFolder(TemporaryFolder basedir, String folder) {
+ try {
+ File newFolder = basedir.newFolder(folder);
+ FileUtil.setPermission(newFolder, FsPermission.getDirDefault());
+ return newFolder;
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to create tmp dir: " +
e.getMessage(), e);
+ }
+ }
+
+ public HiveConf getHiveConf() {
+ return hiveConf;
+ }
+
+ @Override
+ public TemporaryFolder getBaseDir() {
+ return basedir;
+ }
+
+ private void createAndSetFolderProperty(
+ HiveConf.ConfVars var, String folder, HiveConf conf,
TemporaryFolder basedir) {
+ conf.setVar(var, newFolder(basedir, folder).getAbsolutePath());
+ }
+
+ private void createAndSetFolderProperty(
+ String key, String folder, HiveConf conf, TemporaryFolder basedir)
{
+ conf.set(key, newFolder(basedir, folder).getAbsolutePath());
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-site.xml
b/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-site.xml
new file mode 100644
index 000000000..6dbe4c962
--- /dev/null
+++ b/paimon-hive/paimon-hive-connector-common/src/test/resources/hive-site.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <!-- allow integral partition filter pushdown to avoid unstable test -->
+ <property>
+ <name>hive.metastore.integral.jdo.pushdown</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.client.capability.check</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>datanucleus.schema.autoCreateTables</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>datanucleus.schema.autoCreateAll</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>common-key</name>
+ <value>common-val</value>
+ </property>
+
+</configuration>
