yuqi1129 commented on code in PR #8252: URL: https://github.com/apache/gravitino/pull/8252#discussion_r2312980756
########## catalogs/catalog-common/src/main/java/org/apache/gravitino/utils/ClassLoaderResourceCleanerUtils.java: ########## @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.utils; + +import java.lang.reflect.Field; +import java.util.IdentityHashMap; +import java.util.Timer; +import java.util.concurrent.ScheduledExecutorService; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.reflect.MethodUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class to clean up resources related to a specific class loader to prevent memory leaks. + * Gravitino will create a new class loader for each catalog and release it when there exist any + * changes to the catalog. So, it's important to clean up resources related to the class loader to + * prevent memory leaks. + */ +public class ClassLoaderResourceCleanerUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ClassLoaderResourceCleanerUtils.class); + + private ClassLoaderResourceCleanerUtils() {} + + /** + * Close all resources related to the given class loader to prevent memory leaks. + * + * @param classLoader the classloader to be closed + */ + public static void closeClassLoaderResource(ClassLoader classLoader) { + boolean testEnv = System.getenv("GRAVITINO_TEST") != null; + if (testEnv) { + // In test environment, we do not need to clean up class loader related stuff + return; + } + + try { + // Clear statics threads in FileSystem and close all FileSystem instances. + closeStatsDataClearerInFileSystem(classLoader); + + // Stop all threads with the current class loader and clear their threadLocal variables for + // jetty threads that are loaded by the current class loader. + // For example, thread local `threadData` in FileSystem#StatisticsDataCleaner is created + // within jetty thread with the current class loader. However, there are clear by + // `catalog.close` in ForkJoinPool in CaffeineCache, in this case, the thread local variable + // will not be cleared, so we need to clear them manually here. + stopThreadsAndClearThreadLocalVariables(classLoader); + + // Release the LogFactory for the classloader, each classloader has its own LogFactory + // instance. + releaseLogFactoryInCommonLogging(classLoader); + + closeResourceInAWS(classLoader); + + closeResourceInGCP(classLoader); + + closeResourceInAzure(classLoader); + + clearShutdownHooks(classLoader); + } catch (Exception e) { + LOG.warn("Failed to clear resources(threads, thread local variables) in the class loader", e); Review Comment: No, I have caught any possible exceptions in `closeResourceInAWS `. ```java private static void closeResourceInAWS(ClassLoader classLoader) { // For Aws SDK metrics, unregister the metric admin MBean try { Class<?> awsSdkMetricsClass = Class.forName("com.amazonaws.metrics.AwsSdkMetrics", true, classLoader); MethodUtils.invokeStaticMethod(awsSdkMetricsClass, "unregisterMetricAdminMBean"); } catch (Exception e) { LOG.warn( "Failed to unregister AWS SDK metrics admin MBean from class loader {}", classLoader, e); } } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
