Jecarm commented on a change in pull request #2119:
URL: https://github.com/apache/iceberg/pull/2119#discussion_r570773304
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -51,4 +81,918 @@ protected void close(Object client) {
}
}
+
+ private static class MockHiveClientPool extends ClientPool<IMetaStoreClient,
Exception> {
+
+ MockHiveClientPool(int poolSize, Class<? extends Exception> reconnectExc) {
+ super(poolSize, reconnectExc);
+ }
+
+ @Override
+ protected IMetaStoreClient newClient() {
+ return new MockMetaStoreFailureClient();
Review comment:
I refactored the test unit, please review it.
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
Review comment:
i will fix them.
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
Review comment:
the exception of `MetaException` or `TTransportException` for
`getAllFunctions` in this method will not be triggered. If need test
`TTransportException`, we will create a new client In a separate method.
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
Review comment:
not necessary
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
Review comment:
Can we use `testConnectionFailure` test method implementation instead
of `testNewClientFailure` ?
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
Review comment:
like this:
@Test(expected = MetaException.class)
public void testNewClientFailure() throws Exception {
HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
Mockito.doReturn(hmsClient).when(clients).newClient();
Mockito.doThrow(new MetaException("Another meta exception"))
.when(hmsClient).getTables(Mockito.anyString(), Mockito.anyString());
clients.run(client -> client.getTables("default", "t"));
}
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+ Mockito.doThrow(new MetaException("Another meta exception"))
+ .when(hmsClient).getTables(Mockito.anyString(), Mockito.anyString());
+ clients.run(client -> client.getTables("default", "t"));
+ }
+
private static class MockClientPool extends ClientPool<Object, Exception> {
Review comment:
I will clear it
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestClientPool.java
##########
@@ -19,17 +19,83 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestClientPool {
+ HiveClientPool clients;
+
+ @Before
+ public void before() {
+ HiveClientPool clientPool = new HiveClientPool(2, new Configuration());
+ clients = Mockito.spy(clientPool);
+ }
+
+ @After
+ public void after() {
+ clients.close();
+ clients = null;
+ }
+
@Test(expected = RuntimeException.class)
public void testNewClientFailure() throws Exception {
try (MockClientPool pool = new MockClientPool(2, Exception.class)) {
pool.run(Object::toString);
}
}
+ @Test
+ public void testConnectionFailureRestore() throws Exception {
+
+ HiveMetaStoreClient hmsClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(hmsClient).when(clients).newClient();
+
+ // Throwing an exception may trigger the client to reconnect.
+ String metaMessage = "Got exception:
org.apache.thrift.transport.TTransportException";
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllDatabases();
+ Mockito.doThrow(new
MetaException(metaMessage)).when(hmsClient).getAllFunctions();
+
+ // Create a new client when the reconnect method is called.
+ HiveMetaStoreClient newClient = Mockito.mock(HiveMetaStoreClient.class);
+ Mockito.doReturn(newClient).when(clients).reconnect(hmsClient);
+
+ Mockito.doReturn(Lists.newArrayList("db1",
"db2")).when(newClient).getAllDatabases();
+ GetAllFunctionsResponse response = new GetAllFunctionsResponse();
+ response.addToFunctions(
+ new Function("concat", "db1", "classname", "root", PrincipalType.USER,
100, FunctionType.JAVA, null));
+ Mockito.doReturn(response)
+ .when(newClient).getAllFunctions();
+ // The return is OK when the reconnect method is called.
+ Assert.assertEquals(Lists.newArrayList("db1", "db2"),
+ clients.run(client -> client.getAllDatabases()));
+
+ //assert && verify
+ Mockito.verify(clients).reconnect(hmsClient);
+
+ // The return is OK, because the client pool uses a new client
+ Assert.assertEquals(response,
+ clients.run(client -> client.getAllFunctions()));
+
+ Mockito.verify(clients, Mockito.times(1)).reconnect(hmsClient);
+ }
+
+ @Test(expected = MetaException.class)
+ public void testConnectionFailure() throws Exception {
Review comment:
I submitted some code, Please check it out.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
