pvary commented on a change in pull request #2119:
URL: https://github.com/apache/iceberg/pull/2119#discussion_r570253524
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -51,4 +81,918 @@ protected void close(Object client) {
}
}
+
+ private static class MockHiveClientPool extends ClientPool<IMetaStoreClient,
Exception> {
+
+ MockHiveClientPool(int poolSize, Class<? extends Exception> reconnectExc) {
+ super(poolSize, reconnectExc);
+ }
+
+ @Override
+ protected IMetaStoreClient newClient() {
+ return new MockMetaStoreFailureClient();
Review comment:
Even better, we do not have to create a new class for the tests, but we
can use `Mockito.spy()` on `HiveClientPool` and return the different clients on
`newClient` and `reconnect`
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
Review comment:
nit: Please import concrete classes
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
Review comment:
nit: unnecessary line break
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
Review comment:
nit: unnecessary line break
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
Review comment:
nit: unnecessary line break
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
Review comment:
I think it is perfectly ok to test only with a single method with one
type of exception.
Do we want to test failures with `new TTransportException` as well?
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
Review comment:
Should we check?
```
Mockito.verify(clients).reconnect(hmsClient);
Mockito.verify(clients, never()).reconnect(hmsClient);
```
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
Review comment:
If I understand correctly this test the name is slightly misleading.
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+ Mockito.doThrow(new MetaException("Another meta exception"))
+ .when(hmsClient).getTables(Mockito.anyString(), Mockito.anyString());
+ clients.run(client -> client.getTables("default", "t"));
+ }
+
private static class MockClientPool extends ClientPool<Object, Exception> {
Review comment:
Do we still need this class?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
