This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 24413d9f8a HDDS-9002. Container log should write to separate file
(#5053)
24413d9f8a is described below
commit 24413d9f8ae93785d83860a7c25bd8b328162203
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Thu Jul 13 05:31:03 2023 +0200
HDDS-9002. Container log should write to separate file (#5053)
---
.../container/common/utils/ContainerLogger.java | 6 +-
.../TestKeyValueContainerMetadataInspector.java | 5 +-
hadoop-hdds/test-utils/pom.xml | 9 +++
.../org/apache/ozone/test/GenericTestUtils.java | 45 +++++--------
.../java/org/apache/ozone/test/Log4j1Capturer.java | 59 +++++++++++++++++
.../java/org/apache/ozone/test/Log4j2Capturer.java | 75 ++++++++++++++++++++++
...tBackgroundContainerDataScannerIntegration.java | 4 +-
...kgroundContainerMetadataScannerIntegration.java | 4 +-
...estOnDemandContainerDataScannerIntegration.java | 4 +-
9 files changed, 170 insertions(+), 41 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
index 46f8c151e3..6a5c5c1175 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.ozone.container.common.utils;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import static
org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
@@ -40,7 +40,7 @@ public final class ContainerLogger {
@VisibleForTesting
public static final String LOG_NAME = "ContainerLog";
- private static final Logger LOG = LoggerFactory.getLogger(LOG_NAME);
+ private static final Logger LOG = LogManager.getLogger(LOG_NAME);
private static final String FIELD_SEPARATOR = " | ";
private ContainerLogger() { }
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
index f201bf5b48..eaf8939a91 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
@@ -46,6 +46,8 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import static org.apache.ozone.test.GenericTestUtils.toLog4j;
+
/**
* Tests for {@link KeyValueContainerMetadataInspector}.
*/
@@ -499,7 +501,8 @@ public class TestKeyValueContainerMetadataInspector
// parsed as json.
GenericTestUtils.LogCapturer capturer =
GenericTestUtils.LogCapturer.captureLogs(
- KeyValueContainerMetadataInspector.REPORT_LOG, new
PatternLayout());
+ toLog4j(KeyValueContainerMetadataInspector.REPORT_LOG),
+ new PatternLayout());
KeyValueContainerUtil.parseKVContainerData(containerData, getConf());
capturer.stopCapturing();
String output = capturer.getOutput();
diff --git a/hadoop-hdds/test-utils/pom.xml b/hadoop-hdds/test-utils/pom.xml
index 5c5fc0bdbd..e9c8f2b454 100644
--- a/hadoop-hdds/test-utils/pom.xml
+++ b/hadoop-hdds/test-utils/pom.xml
@@ -71,6 +71,15 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.jacoco</groupId>
<artifactId>org.jacoco.core</artifactId>
diff --git
a/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/GenericTestUtils.java
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/GenericTestUtils.java
index b2215f46ed..9cd97eaabb 100644
---
a/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/GenericTestUtils.java
+++
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/GenericTestUtils.java
@@ -34,13 +34,10 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.log4j.Appender;
import org.apache.log4j.Layout;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.WriterAppender;
import org.junit.Assert;
import org.mockito.Mockito;
import java.lang.reflect.Field;
@@ -312,48 +309,40 @@ public abstract class GenericTestUtils {
/**
* Class to capture logs for doing assertions.
*/
- public static final class LogCapturer {
- private StringWriter sw = new StringWriter();
- private WriterAppender appender;
- private Logger logger;
+ public abstract static class LogCapturer {
+ private final StringWriter sw = new StringWriter();
- public static LogCapturer captureLogs(org.slf4j.Logger logger) {
- return new LogCapturer(toLog4j(logger), getDefaultLayout());
+ public static LogCapturer captureLogs(Logger logger) {
+ return new Log4j1Capturer(logger);
}
- public static LogCapturer captureLogs(org.slf4j.Logger logger,
- Layout layout) {
- return new LogCapturer(toLog4j(logger), layout);
+ public static LogCapturer captureLogs(Logger logger, Layout layout) {
+ return new Log4j1Capturer(logger, layout);
}
- private static Layout getDefaultLayout() {
- Appender defaultAppender = Logger.getRootLogger().getAppender("stdout");
- if (defaultAppender == null) {
- defaultAppender = Logger.getRootLogger().getAppender("console");
- }
- return (defaultAppender == null) ? new PatternLayout() :
- defaultAppender.getLayout();
+ public static LogCapturer captureLogs(org.slf4j.Logger logger) {
+ return new Log4j1Capturer(toLog4j(logger));
}
- private LogCapturer(Logger logger, Layout layout) {
- this.logger = logger;
- this.appender = new WriterAppender(layout, sw);
- logger.addAppender(this.appender);
+ // TODO: let Log4j2Capturer capture only specific logger's logs
+ public static LogCapturer log4j2(String ignoredLoggerName) {
+ return Log4j2Capturer.getInstance();
}
public String getOutput() {
- return sw.toString();
+ return writer().toString();
}
- public void stopCapturing() {
- logger.removeAppender(appender);
+ public abstract void stopCapturing();
+
+ protected StringWriter writer() {
+ return sw;
}
public void clearOutput() {
- sw.getBuffer().setLength(0);
+ writer().getBuffer().setLength(0);
}
}
-
@Deprecated
public static Logger toLog4j(org.slf4j.Logger logger) {
return LogManager.getLogger(logger.getName());
diff --git
a/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j1Capturer.java
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j1Capturer.java
new file mode 100644
index 0000000000..58f7088a53
--- /dev/null
+++
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j1Capturer.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozone.test;
+
+import org.apache.log4j.Appender;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.apache.log4j.WriterAppender;
+
+/**
+ * Capture Log4j1 logs.
+ */
+class Log4j1Capturer extends GenericTestUtils.LogCapturer {
+ private final Appender appender;
+ private final Logger logger;
+
+ private static Layout getDefaultLayout() {
+ Logger rootLogger = Logger.getRootLogger();
+
+ Appender defaultAppender = rootLogger.getAppender("stdout");
+ if (defaultAppender == null) {
+ defaultAppender = rootLogger.getAppender("console");
+ }
+
+ return defaultAppender == null
+ ? new PatternLayout()
+ : defaultAppender.getLayout();
+ }
+
+ Log4j1Capturer(Logger logger) {
+ this(logger, getDefaultLayout());
+ }
+
+ Log4j1Capturer(Logger logger, Layout layout) {
+ this.logger = logger;
+ this.appender = new WriterAppender(layout, writer());
+ logger.addAppender(appender);
+ }
+
+ public void stopCapturing() {
+ logger.removeAppender(appender);
+ }
+}
diff --git
a/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j2Capturer.java
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j2Capturer.java
new file mode 100644
index 0000000000..f00b3291bf
--- /dev/null
+++
b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/Log4j2Capturer.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ozone.test;
+
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.WriterAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import java.io.Writer;
+
+/**
+ * Capture Log4j2 logs.
+ */
+final class Log4j2Capturer extends GenericTestUtils.LogCapturer {
+ private static final String APPENDER_NAME = "capture";
+
+ private static final Log4j2Capturer INSTANCE = new Log4j2Capturer();
+
+ public static Log4j2Capturer getInstance() {
+ return INSTANCE;
+ }
+
+ private Log4j2Capturer() {
+ addAppender(writer());
+ }
+
+ public void stopCapturing() {
+ final LoggerContext context = LoggerContext.getContext(false);
+ final Configuration config = context.getConfiguration();
+ removeAppender(config);
+ }
+
+ private void addAppender(final Writer writer) {
+ final LoggerContext context = LoggerContext.getContext(false);
+ final Configuration config = context.getConfiguration();
+ final PatternLayout layout = PatternLayout.createDefaultLayout(config);
+ final Appender appender = WriterAppender.createAppender(layout, null,
+ writer, APPENDER_NAME, false, true);
+ appender.start();
+ config.addAppender(appender);
+ addAppender(appender, config);
+ }
+
+ private void addAppender(Appender appender, Configuration config) {
+ for (final LoggerConfig loggerConfig : config.getLoggers().values()) {
+ loggerConfig.addAppender(appender, null, null);
+ }
+ config.getRootLogger().addAppender(appender, null, null);
+ }
+
+ private void removeAppender(final Configuration config) {
+ for (final LoggerConfig loggerConfig : config.getLoggers().values()) {
+ loggerConfig.removeAppender(APPENDER_NAME);
+ }
+ config.getRootLogger().removeAppender(APPENDER_NAME);
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java
index d8eb73fb0d..79a60d7968 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerDataScannerIntegration.java
@@ -31,7 +31,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.concurrent.TimeUnit;
@@ -73,8 +72,7 @@ public class TestBackgroundContainerDataScannerIntegration
public TestBackgroundContainerDataScannerIntegration(
ContainerCorruptions corruption) {
this.corruption = corruption;
- logCapturer = LogCapturer.captureLogs(
- LoggerFactory.getLogger(ContainerLogger.LOG_NAME));
+ logCapturer =
GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME);
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java
index 0713aeacf1..ee0e852f2d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestBackgroundContainerMetadataScannerIntegration.java
@@ -31,7 +31,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.util.Collection;
@@ -84,8 +83,7 @@ public class TestBackgroundContainerMetadataScannerIntegration
public TestBackgroundContainerMetadataScannerIntegration(
ContainerCorruptions corruption) {
this.corruption = corruption;
- logCapturer = GenericTestUtils.LogCapturer.captureLogs(
- LoggerFactory.getLogger(ContainerLogger.LOG_NAME));
+ logCapturer =
GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME);
}
/**
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
index 2c60c577f0..1430e4f965 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
@@ -30,7 +30,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
-import org.slf4j.LoggerFactory;
import java.util.Collection;
@@ -89,8 +88,7 @@ public class TestOnDemandContainerDataScannerIntegration
public TestOnDemandContainerDataScannerIntegration(
ContainerCorruptions corruption) {
this.corruption = corruption;
- logCapturer = GenericTestUtils.LogCapturer.captureLogs(
- LoggerFactory.getLogger(ContainerLogger.LOG_NAME));
+ logCapturer =
GenericTestUtils.LogCapturer.log4j2(ContainerLogger.LOG_NAME);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
