This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 89631dc8c43 HIVE-29470: Add unit test to verify commonLocalResources
after opening a Tez session (#6338)
89631dc8c43 is described below
commit 89631dc8c43ee951017fe6ae3ed7c5286f90be8d
Author: kokila-19 <[email protected]>
AuthorDate: Wed Mar 4 16:31:20 2026 +0530
HIVE-29470: Add unit test to verify commonLocalResources after opening a
Tez session (#6338)
Ensures buildCommonLocalResources populates the map with the hive-exec app
jar and localized resources so regressions are caught if resource
registration is removed.
---
.../hadoop/hive/ql/exec/tez/TezSessionState.java | 21 ++++++---
.../hive/ql/exec/tez/TestTezSessionState.java | 52 +++++++++++++++++++---
2 files changed, 62 insertions(+), 11 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 41e01b17feb..1b6965018a8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -302,6 +302,20 @@ protected void openInternal(String[]
additionalFilesNotFromConf,
}
}
+ /**
+ * Builds the map of common local resources (app jar + localized resources)
for the Tez session.
+ * Extracted for testability to ensure resources are properly registered.
+ */
+ @VisibleForTesting
+ Map<String, LocalResource> buildCommonLocalResources() {
+ final Map<String, LocalResource> commonLocalResources = new HashMap<>();
+ commonLocalResources.put(DagUtils.getBaseName(appJarLr), appJarLr);
+ for (LocalResource lr : this.resources.localizedResources) {
+ commonLocalResources.put(DagUtils.getBaseName(lr), lr);
+ }
+ return commonLocalResources;
+ }
+
/**
* Opens a Tez session without performing a complete rollback/cleanup on
failure.
*
@@ -318,14 +332,9 @@ protected void openInternal(String[]
additionalFilesNotFromConf,
*/
@VisibleForTesting
void openInternalUnsafe(boolean isAsync, LogHelper console) throws
TezException, IOException {
- final Map<String, LocalResource> commonLocalResources = new HashMap<>();
+ final Map<String, LocalResource> commonLocalResources =
buildCommonLocalResources();
final boolean llapMode = "llap".equalsIgnoreCase(HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_EXECUTION_MODE));
- commonLocalResources.put(DagUtils.getBaseName(appJarLr), appJarLr);
- for (LocalResource lr : this.resources.localizedResources) {
- commonLocalResources.put(DagUtils.getBaseName(lr), lr);
- }
-
if (llapMode) {
// localize llap client jars
addJarLRByClass(LlapTaskSchedulerService.class, commonLocalResources);
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
index 62e0d0cd27c..bafd0e547dc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestTezSessionState.java
@@ -21,6 +21,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -36,6 +37,13 @@
public class TestTezSessionState {
private static final Logger LOG =
LoggerFactory.getLogger(TestTezSessionState.class.getName());
+ private static SessionState createSessionState() {
+ HiveConf hiveConf = new HiveConfForTest(TestTezSessionState.class);
+ hiveConf.set("hive.security.authorization.manager",
+
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory");
+ return SessionState.start(hiveConf);
+ }
+
@Test
public void testSymlinkedLocalFilesAreLocalizedOnce() throws Exception {
Path jarPath = Files.createTempFile("jar", "");
@@ -60,14 +68,12 @@ public void testSymlinkedLocalFilesAreLocalizedOnce()
throws Exception {
@Test
public void testScratchDirDeletedInTheEventOfExceptionWhileOpeningSession()
throws Exception {
- HiveConf hiveConf = new HiveConfForTest(getClass());
- hiveConf.set("hive.security.authorization.manager",
-
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory");
- SessionState.start(hiveConf);
+ SessionState ss = createSessionState();
+ HiveConf hiveConf = ss.getConf();
final AtomicReference<String> scratchDirPath = new AtomicReference<>();
- TezSessionState sessionState = new
TezSessionState(SessionState.get().getSessionId(), hiveConf) {
+ TezSessionState sessionState = new TezSessionState(ss.getSessionId(),
hiveConf) {
@Override
void openInternalUnsafe(boolean isAsync, SessionState.LogHelper console)
throws TezException, IOException {
super.openInternalUnsafe(isAsync, console);
@@ -90,4 +96,40 @@ void openInternalUnsafe(boolean isAsync,
SessionState.LogHelper console) throws
Assert.assertFalse("Scratch dir is not supposed to exist after cleanup: "
+ scratchDirPath.get(),
Files.exists(Paths.get(scratchDirPath.get())));
}
+
+ /**
+ * Tests whether commonLocalResources is populated with app jar and
localized resources when opening
+ * a Tez session.
+ */
+ @Test
+ public void testCommonLocalResourcesPopulatedOnSessionOpen() throws
Exception {
+ Path jarPath = Files.createTempFile("test-jar", ".jar");
+ Files.write(jarPath, "testCommonLocalResourcesPopulated".getBytes(),
StandardOpenOption.APPEND);
+
+ SessionState ss = createSessionState();
+ HiveConf hiveConf = ss.getConf();
+
+ TezSessionState.HiveResources resources =
+ new TezSessionState.HiveResources(new
org.apache.hadoop.fs.Path("/tmp"));
+
+ TezSessionState tempSession = new TezSessionState(ss.getSessionId(),
hiveConf);
+
+ LocalResource localizedLr =
tempSession.createJarLocalResource(jarPath.toUri().toString());
+ resources.localizedResources.add(localizedLr);
+
+ final TezSessionState sessionStateForTest = new
TezSessionState(ss.getSessionId(), hiveConf) {
+ @Override
+ void openInternalUnsafe(boolean isAsync, SessionState.LogHelper console)
{
+ Map<String, LocalResource> commonLocalResources =
buildCommonLocalResources();
+ Assert.assertEquals("commonLocalResources must contain exactly 2 jars
(hive-exec app jar + localized test jar)",
+ 2, commonLocalResources.size());
+ Assert.assertTrue("commonLocalResources must contain the hive-exec app
jar",
+ commonLocalResources.keySet().stream().anyMatch(k ->
k.contains("hive-exec")));
+ Assert.assertTrue("commonLocalResources must contain the added
localized test jar",
+
commonLocalResources.containsKey(DagUtils.getBaseName(localizedLr)));
+ }
+ };
+
+ sessionStateForTest.open(resources);
+ }
}
\ No newline at end of file
