danhuawang opened a new issue, #9782:
URL: https://github.com/apache/gravitino/issues/9782
### Version
main branch
### Describe what's wrong
I scale the IRC to 2 instance, then create tables with 10 users
concurrently, It occasionally pop the following error:
```
================================================================================
ERROR REPORT - DETAILED REQUEST/RESPONSE INFORMATION
================================================================================
4 occurrences | POST
http://10.50.0.7:9001/iceberg/v1/iceberg/namespaces/dml25/tables: 500
--------------------------------------------------------------------------------
Example 1:
Time: 11:00:25
URL: http://10.50.0.7:9001/iceberg/v1/iceberg/namespaces/dml25/tables
Method: POST
Status Code: 500
Exception: HTTP 500 Error
Request Body:
{
"name": "test_tb_0053860",
"location": "gs://strato-iceberg/IRCperform/dml25/test_tb_0053860",
"schema": {
"type": "struct",
"schema-id": 0,
"fields": [
{
"id": 1,
"name": "id",
"required": true,
"type": "int",
"write-default": 1
},
{
... (request body truncated)
Response Body:
{
"error": {
"message": "Cannot get a client from a closed pool",
"type": "IllegalStateException",
"code": 500,
"stack": [
"java.lang.IllegalStateException: Cannot get a client from a
closed pool",
"\tat
org.apache.iceberg.relocated.com.google.common.base.Preconditions.checkState(Preconditions.java:513)",
"\tat
org.apache.iceberg.ClientPoolImpl.get(ClientPoolImpl.java:136)",
"\tat
org.apache.iceberg.ClientPoolImpl.run(ClientPoolImpl.java:70)",
"\tat
org.apache.iceberg.ClientPoolImpl.run(ClientPoolImpl.java:65)",
"\tat
org.apache.iceberg.jdbc.JdbcUtil.tableOrView(JdbcUtil.java:602)",
"\tat
org.apache.iceberg.jdbc.JdbcUtil.loadTable(JdbcUtil.java:641)",
"\tat
org.apache.iceberg.jdbc.JdbcTableOperations.doCommit(JdbcTableOperations.java:109)",
"\tat
org.apache.iceberg.BaseMetastoreTableOperations.commit(BaseMetastoreTableOperations.java:126)",
"\tat
org.apache.iceberg.BaseMetastoreCatalog$BaseMetastoreCatalogTableBuilder.create(BaseMetastoreCatalog.java:201)",
"\tat
org.apache.iceberg.rest.CatalogHandlers.createTable(CatalogHandlers.java:281)",
"\tat
org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper.createTable(IcebergCatalogWrapper.java:175)",
"\tat
org.apache.gravitino.iceberg.service.CatalogWrapperForREST.createTable(CatalogWrapperForREST.java:113)",
"\tat
org.apache.gravitino.iceberg.service.dispatcher.IcebergTableOperationExecutor.createTable(IcebergTableOperationExecutor.java:95)",
"\tat
org.apache.gravitino.iceberg.service.dispatcher.IcebergTableHookDispatcher.createTable(IcebergTableHookDispatcher.java:59)",
"\tat
org.apache.gravitino.iceberg.service.dispatcher.IcebergTableEventDispatcher.createTable(IcebergTableEventDispatcher.java:101)",
"\tat
org.apache.gravitino.iceberg.service.rest.IcebergTableOperations.lambda$createTable$1(IcebergTableOperations.java:175)",
"\tat
java.base/java.security.AccessController.doPrivileged(AccessController.java:712)",
"\tat
java.base/javax.security.auth.Subject.doAs(Subject.java:439)",
"\tat
org.apache.gravitino.utils.PrincipalUtils.doAs(PrincipalUtils.java:44)",
"\tat
org.apache.gravitino.server.web.Utils.doAs(Utils.java:202)",
"\tat
org.apache.gravitino.iceberg.service.rest.IcebergTableOperations.createTable(IcebergTableOperations.java:169)",
"\tat
org.apache.gravitino.iceberg.service.rest.IcebergTableOperations_$$_jvst14d_0._d2createTable(IcebergTableOperations_$$_jvst14d_0.java)",
"\tat
jdk.internal.reflect.GeneratedMethodAccessor93.invoke(Unknown Source)",
"\tat
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)",
"\tat
java.base/java.lang.reflect.Method.invoke(Method.java:569)",
"\tat
org.glassfish.hk2.utilities.reflection.ReflectionHelper.invoke(ReflectionHelper.java:1268)",
"\tat
org.jvnet.hk2.internal.MethodInterceptorHandler$MethodInvocationImpl.proceed(MethodInterceptorHandler.java:164)",
"\tat
org.apache.gravitino.server.web.filter.BaseMetadataAuthorizationMethodInterceptor.invoke(BaseMetadataAuthorizationMethodInterceptor.java:203)",
"\tat
org.jvnet.hk2.internal.MethodInterceptorHandler.invoke(MethodInterceptorHandler.java:97)",
"\tat
org.apache.gravitino.iceberg.service.rest.IcebergTableOperations_$$_jvst14d_0.createTable(IcebergTableOperations_$$_jvst14d_0.java)",
"\tat
jdk.internal.reflect.GeneratedMethodAccessor92.invoke(Unknown Source)",
"\tat
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)",
"\tat
java.base/java.lang.reflect.Method.invoke(Method.java:569)",
"\tat
org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52)",
"\tat
org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:146)",
"\tat
org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:189)",
"\tat
org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:176)",
"\tat
org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:93)",
"\tat
org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:478)",
"\tat
org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:400)",
"\tat
org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:81)",
"\tat
org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:256)",
"\tat
org.glassfish.jersey.internal.Errors$1.call(Errors.java:248)",
"\tat
org.glassfish.jersey.internal.Errors$1.call(Errors.java:244)",
"\tat
org.glassfish.jersey.internal.Errors.process(Errors.java:292)",
"\tat
org.glassfish.jersey.internal.Errors.process(Errors.java:274)",
"\tat
org.glassfish.jersey.internal.Errors.process(Errors.java:244)",
"\tat
org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265)",
"\tat
org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:235)",
"\tat
org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:684)",
"\tat
org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:394)",
"\tat
org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:346)",
"\tat
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:358)",
"\tat
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:311)",
"\tat
org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:205)",
"\tat
org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)",
"\tat
org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1656)",
"\tat
org.apache.gravitino.server.authentication.AuthenticationFilter.lambda$doFilter$0(AuthenticationFilter.java:89)",
"\tat
java.base/java.security.AccessController.doPrivileged(AccessController.java:712)",
"\tat
java.base/javax.security.auth.Subject.doAs(Subject.java:439)",
"\tat
org.apache.gravitino.utils.PrincipalUtils.doAs(PrincipalUtils.java:44)",
"\tat
org.apache.gravitino.server.authentication.AuthenticationFilter.doFilter(AuthenticationFilter.java:86)",
"\tat
org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)",
"\tat
org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1626)",
"\tat
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:552)",
"\tat
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)",
"\tat
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440)",
"\tat
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)",
"\tat
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:505)",
"\tat
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)",
"\tat
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355)",
"\tat
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)",
"\tat
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)",
"\tat
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)",
"\tat org.eclipse.jetty.server.Server.handle(Server.java:516)",
"\tat
org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487)",
"\tat
org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732)",
"\tat
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479)",
"\tat
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)",
"\tat
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)",
"\tat
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)",
"\tat
org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)",
"\tat
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)",
"\tat
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)",
"\tat
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)",
"\tat
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)",
"\tat
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409)",
"\tat
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)",
"\tat
org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)",
"\tat java.base/java.lang.Thread.run(Thread.java:840)"
]
}
}
```
### Error message and/or stacktrace
see above
### How to reproduce
1. Scale the IRC instance number to 2, but they shared the same MySQL
backend storage
2. Set up a load balancer for access
3. Run the test command ` locust --headless --users 10 --spawn-rate 1 --host
http://10.50.0.7:9001 --schema dml25 --tags create --run-time 5m --csv=report
-f locustfile_db_credentialvending_users_latest.py -L DEBUG`
### Additional context
_No response_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
