This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0eea2a36b44 HIVE-27911 : Drop database query failing with Invalid ACL
Exception (#4901) (Kirti Ruge reviewed by Laszlo Bodor)
0eea2a36b44 is described below
commit 0eea2a36b440e7c017e0a1a565ebdfb64c723abc
Author: rkirtir <[email protected]>
AuthorDate: Mon Jan 8 12:48:50 2024 +0530
HIVE-27911 : Drop database query failing with Invalid ACL Exception (#4901)
(Kirti Ruge reviewed by Laszlo Bodor)
---
.../registry/impl/LlapZookeeperRegistryImpl.java | 10 +++++++
.../hadoop/hive/registry/impl/ZkRegistryBase.java | 31 +++++++++++++++++++++-
.../impl/TestLlapZookeeperRegistryImpl.java | 12 +++++++++
3 files changed, 52 insertions(+), 1 deletion(-)
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
index 25cb89a3756..1c4df16b764 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java
@@ -168,6 +168,16 @@ public class LlapZookeeperRegistryImpl
String uniqueId = UNIQUE_ID.toString();
long znodeCreationTimeout = 120;
+ /* When no LLAP executors are running, the getInstances()/
llapRegistryService.getInstances() method results in
+ InvalidACL exception when Hive tries to evict the cache in DROP
DATABASE/TABLE
+
+ As ContainerManager of zookeeper makes sure to cleanup znodes
periodically, once query-coordinator zookeeper
+ client session is terminated,entire path will get deleted sooner or
later. This results in InvalidACL exception
+
+ PersistentNode created on server will be preserved through restarts of
query executor and makes sure
+ proactive eviction call of DROP DATABASE/ DROP TABLE go through
+ successfully. */
+ ensurePersistentNodePath(daemonZkRecord);
initializeWithoutRegisteringInternal();
// Create a znode under the rootNamespace parent for this instance of the
server
try {
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
index 9da200f2e92..f6a52cd3b2d 100644
---
a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
+++
b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
@@ -29,6 +29,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.api.ACLProvider;
import org.apache.curator.framework.imps.CuratorFrameworkState;
@@ -55,6 +57,7 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.InvalidACLException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.apache.zookeeper.ZooDefs;
@@ -114,6 +117,8 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
private PersistentNode znode;
private String znodePath; // unique identity for this instance
+ final String namespace;
+
private PathChildrenCache instancesCache; // Created on demand.
/** Local hostname. */
@@ -160,7 +165,7 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
this.stateChangeListeners = new HashSet<>();
this.pathToInstanceCache = new ConcurrentHashMap<>();
this.nodeToInstanceCache = new ConcurrentHashMap<>();
- final String namespace = getRootNamespace(conf, rootNs, nsPrefix);
+ this.namespace = getRootNamespace(conf, rootNs, nsPrefix);
ACLProvider aclProvider;
// get acl provider for most outer path that is non-null
if (userPathPrefix == null) {
@@ -353,6 +358,30 @@ public abstract class ZkRegistryBase<InstanceType extends
ServiceInstance> {
}
}
+ @VisibleForTesting
+ public String getPersistentNodePath() {
+ return "/" + PATH_JOINER.join(namespace,
StringUtils.substringBetween(workersPath, "/", "/"), "pnode0");
+ }
+
+ protected void ensurePersistentNodePath(ServiceRecord srv) throws
IOException {
+ String pNodePath = getPersistentNodePath();
+ try {
+ LOG.info("Check if persistent node path {} exists, create if not",
pNodePath);
+ if (zooKeeperClient.checkExists().forPath(pNodePath) == null) {
+
zooKeeperClient.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT)
+ .forPath(pNodePath, encoder.toBytes(srv));
+ LOG.info("Created persistent path at: {}", pNodePath);
+ }
+ } catch (Exception e) {
+ // throw exception if it is other than NODEEXISTS.
+ if (!(e instanceof KeeperException) || ((KeeperException) e).code() !=
KeeperException.Code.NODEEXISTS) {
+ LOG.error("Unable to create a persistent znode for this server
instance", e);
+ throw new IOException(e);
+ } else {
+ LOG.debug("Ignoring KeeperException while ensuring path as the parent
node {} already exists.", pNodePath);
+ }
+ }
+ }
final protected void initializeWithoutRegisteringInternal() throws
IOException {
// Create a znode under the rootNamespace parent for this instance of the
server
diff --git
a/llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
b/llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
index 50799f2f5ca..7b227d4356b 100644
---
a/llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
+++
b/llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
import org.apache.hadoop.hive.registry.ServiceInstanceSet;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -47,6 +48,7 @@ public class TestLlapZookeeperRegistryImpl {
private CuratorFramework curatorFramework;
private TestingServer server;
+ private final static String NAMESPACE_PREFIX = "llap-";
@Before
public void setUp() throws Exception {
@@ -124,6 +126,16 @@ public class TestLlapZookeeperRegistryImpl {
attributes.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS));
}
+ @Test
+ public void testPersistentNodePath() {
+ String llapRootNameSpace = "/" +
LlapZookeeperRegistryImpl.getRootNamespace(hiveConf,
+ HiveConf.getVar(hiveConf,
HiveConf.ConfVars.LLAP_ZK_REGISTRY_NAMESPACE), NAMESPACE_PREFIX);
+ String persistentNodeName = "/pnode0";
+
+ Assert.assertEquals(llapRootNameSpace + "/user-" +
System.getProperty("user.name") + persistentNodeName,
+ registry.getPersistentNodePath());
+ }
+
static <T> void trySetMock(Object o, String field, T value) {
try {
Field fieldToChange =
Arrays.stream(FieldUtils.getAllFields(o.getClass()))
