This is an automated email from the ASF dual-hosted git repository.
jhelou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 4c71ca2384 [devscout] upgrade logback and crowdsec
4c71ca2384 is described below
commit 4c71ca23844907353b61ce5c121e91fb15ccd4e1
Author: Jean Helou <j...@xn--gml-cma.com>
AuthorDate: Sat Dec 14 15:44:15 2024 +0100
[devscout] upgrade logback and crowdsec
This commit drops the unmaintained logback-classic dependecy
and upgrades logback to the latest version.
Logback-classic has a native JsonEncoder (already in the
original version). The latest version allows to have the
formatted message and to enable/disable any top level field\
by configuration.
This allows using logback's own JSON encoder and get rid ot logback-contrib
which is unmaintained to output json ( the repository is archived and the
dependencies are quite outdated)
A side effect was that the format of the Json output changes slightly, I
adjusted the samplec configurations for the crowdsec module.
One of the change is a higher resolution for the timestamp, which required
raising crowdsec to 1.6.x.
---
docs/modules/servers/partials/operate/logging.adoc | 12 ++++
pom.xml | 7 +--
.../protocols/netty/ProtocolMDCContextFactory.java | 1 -
server/apps/cassandra-app/pom.xml | 4 --
.../helm-chart/james/configs/logback.xml | 30 ++++-----
server/apps/distributed-app/pom.xml | 4 --
server/apps/distributed-pop3-app/pom.xml | 4 --
server/apps/jpa-app/pom.xml | 4 --
server/apps/jpa-smtp-app/pom.xml | 4 --
server/apps/memory-app/pom.xml | 4 --
server/apps/scaling-pulsar-smtp/pom.xml | 4 --
.../crowdsec/sample-configuration/logback.xml | 71 +++++++++-------------
.../apache/james/crowdsec/CrowdsecExtension.java | 9 ++-
.../resources/crowdsec/parsers/syslog-logs.yaml | 6 +-
.../crowdsec/src/test/resources/logback-test.xml | 30 +++++----
15 files changed, 87 insertions(+), 107 deletions(-)
diff --git a/docs/modules/servers/partials/operate/logging.adoc
b/docs/modules/servers/partials/operate/logging.adoc
index e8da4c3a4e..0d21945b76 100644
--- a/docs/modules/servers/partials/operate/logging.adoc
+++ b/docs/modules/servers/partials/operate/logging.adoc
@@ -25,6 +25,18 @@ and link:https://docs.fluentbit.io/[FluentBit] as centralize
logging.
Information about logback configuration can be found
link:http://logback.qos.ch/manual/configuration.html[here].
+It is possible to configure a logback appender to expose structured logging
+as JSON using logback's `JsonEncoder` like in the following example:
+
+```
+<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.JsonEncoder">
+ <withFormattedMessage>true</withFormattedMessage>
+ </encoder>
+ <immediateFlush>false</immediateFlush>
+</appender>
+```
+
Note that dedicated formatters can be added to your docker containers :
- Add the corresponding JAR onto the `/root/libs` folder of docker container
diff --git a/pom.xml b/pom.xml
index 12a9e45924..f2622cbd08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -651,7 +651,7 @@
<lucene.version>9.11.1</lucene.version>
<jasypt.version>1.9.3</jasypt.version>
<guice.version>7.0.0</guice.version>
- <logback.version>1.4.14</logback.version>
+ <logback.version>1.5.12</logback.version>
<tink.version>1.9.0</tink.version>
<lettuce.core.version>6.3.2.RELEASE</lettuce.core.version>
<io.micrometer.core.version>1.13.6</io.micrometer.core.version>
@@ -2076,11 +2076,6 @@
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- <version>0.1.5</version>
- </dependency>
<dependency>
<groupId>com.beetstra.jutf7</groupId>
<artifactId>jutf7</artifactId>
diff --git
a/protocols/netty/src/main/java/org/apache/james/protocols/netty/ProtocolMDCContextFactory.java
b/protocols/netty/src/main/java/org/apache/james/protocols/netty/ProtocolMDCContextFactory.java
index d7adfde75f..096fb1e2fc 100644
---
a/protocols/netty/src/main/java/org/apache/james/protocols/netty/ProtocolMDCContextFactory.java
+++
b/protocols/netty/src/main/java/org/apache/james/protocols/netty/ProtocolMDCContextFactory.java
@@ -96,7 +96,6 @@ public interface ProtocolMDCContextFactory {
static MDCBuilder forSession(ProtocolSession protocolSession) {
return MDCBuilder.create()
.addToContext(MDCBuilder.SESSION_ID,
protocolSession.getSessionID())
- .addToContext(MDCBuilder.CHARSET,
protocolSession.getCharset().displayName())
.addToContextIfPresent(MDCBuilder.USER,
Optional.ofNullable(protocolSession.getUsername()).map(Username::asString));
}
diff --git a/server/apps/cassandra-app/pom.xml
b/server/apps/cassandra-app/pom.xml
index a78b343daf..36ddca71d4 100644
--- a/server/apps/cassandra-app/pom.xml
+++ b/server/apps/cassandra-app/pom.xml
@@ -247,10 +247,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>com.linagora</groupId>
<artifactId>logback-elasticsearch-appender</artifactId>
diff --git a/server/apps/distributed-app/helm-chart/james/configs/logback.xml
b/server/apps/distributed-app/helm-chart/james/configs/logback.xml
index 981c842472..8f0f133b9e 100644
--- a/server/apps/distributed-app/helm-chart/james/configs/logback.xml
+++ b/server/apps/distributed-app/helm-chart/james/configs/logback.xml
@@ -1,20 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
under the License.
-->
diff --git a/server/apps/distributed-app/pom.xml
b/server/apps/distributed-app/pom.xml
index 95972160e1..7e98b1af44 100644
--- a/server/apps/distributed-app/pom.xml
+++ b/server/apps/distributed-app/pom.xml
@@ -308,10 +308,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>com.linagora</groupId>
<artifactId>logback-elasticsearch-appender</artifactId>
diff --git a/server/apps/distributed-pop3-app/pom.xml
b/server/apps/distributed-pop3-app/pom.xml
index 3cff0f9130..44b37d9ef0 100644
--- a/server/apps/distributed-pop3-app/pom.xml
+++ b/server/apps/distributed-pop3-app/pom.xml
@@ -299,10 +299,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>com.linagora</groupId>
<artifactId>logback-elasticsearch-appender</artifactId>
diff --git a/server/apps/jpa-app/pom.xml b/server/apps/jpa-app/pom.xml
index 52d9a102aa..77ff003e3e 100644
--- a/server/apps/jpa-app/pom.xml
+++ b/server/apps/jpa-app/pom.xml
@@ -200,10 +200,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>com.linagora</groupId>
<artifactId>logback-elasticsearch-appender</artifactId>
diff --git a/server/apps/jpa-smtp-app/pom.xml b/server/apps/jpa-smtp-app/pom.xml
index 10998e1d54..6b726ea5ad 100644
--- a/server/apps/jpa-smtp-app/pom.xml
+++ b/server/apps/jpa-smtp-app/pom.xml
@@ -148,10 +148,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>com.linagora</groupId>
<artifactId>logback-elasticsearch-appender</artifactId>
diff --git a/server/apps/memory-app/pom.xml b/server/apps/memory-app/pom.xml
index 1ed803261c..42ec3ee4a9 100644
--- a/server/apps/memory-app/pom.xml
+++ b/server/apps/memory-app/pom.xml
@@ -218,10 +218,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
diff --git a/server/apps/scaling-pulsar-smtp/pom.xml
b/server/apps/scaling-pulsar-smtp/pom.xml
index 77e3559318..0368dd56a1 100644
--- a/server/apps/scaling-pulsar-smtp/pom.xml
+++ b/server/apps/scaling-pulsar-smtp/pom.xml
@@ -158,10 +158,6 @@
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
- <dependency>
- <groupId>ch.qos.logback.contrib</groupId>
- <artifactId>logback-json-classic</artifactId>
- </dependency>
<dependency>
<groupId>io.rest-assured</groupId>
<artifactId>rest-assured</artifactId>
diff --git a/third-party/crowdsec/sample-configuration/logback.xml
b/third-party/crowdsec/sample-configuration/logback.xml
index af4b9feb1d..11b3556f24 100644
--- a/third-party/crowdsec/sample-configuration/logback.xml
+++ b/third-party/crowdsec/sample-configuration/logback.xml
@@ -1,49 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
- -->
-
-<configuration>
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
- <contextListener
class="ch.qos.logback.classic.jul.LevelChangePropagator">
- <resetJUL>true</resetJUL>
- </contextListener>
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
- <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
- <encoder
class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
- <layout
class="ch.qos.logback.contrib.json.classic.JsonLayout">
-
<timestampFormat>yyyy-MM-dd'T'HH:mm:ss.SSSX</timestampFormat>
-
<timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
+<configuration>
- <!-- Importance for handling multiple lines
log -->
- <appendLineSeparator>true</appendLineSeparator>
+ <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+ <resetJUL>true</resetJUL>
+ </contextListener>
- <jsonFormatter
class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter">
- <prettyPrint>false</prettyPrint>
- </jsonFormatter>
- </layout>
- </encoder>
- <immediateFlush>false</immediateFlush>
- </appender>
- <root level="WARN">
- <appender-ref ref="CONSOLE" />
- </root>
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder class="ch.qos.logback.classic.encoder.JsonEncoder">
+ <withFormattedMessage>true</withFormattedMessage>
+ </encoder>
+ <immediateFlush>false</immediateFlush>
+ </appender>
+ <root level="WARN">
+ <appender-ref ref="CONSOLE" />
+ </root>
- <logger name="org.apache.james" level="INFO" />
-</configuration>
+ <logger name="org.apache.james" level="INFO" />
+</configuration>
\ No newline at end of file
diff --git
a/third-party/crowdsec/src/test/java/org/apache/james/crowdsec/CrowdsecExtension.java
b/third-party/crowdsec/src/test/java/org/apache/james/crowdsec/CrowdsecExtension.java
index eb834b57e5..af93e1a4f1 100644
---
a/third-party/crowdsec/src/test/java/org/apache/james/crowdsec/CrowdsecExtension.java
+++
b/third-party/crowdsec/src/test/java/org/apache/james/crowdsec/CrowdsecExtension.java
@@ -34,8 +34,11 @@ import
org.apache.james.crowdsec.client.CrowdsecClientConfiguration;
import org.apache.james.crowdsec.client.CrowdsecHttpClient;
import org.apache.james.util.docker.RateLimiters;
import org.junit.jupiter.api.extension.ExtensionContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testcontainers.containers.BindMode;
import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
import org.testcontainers.utility.MountableFile;
@@ -47,7 +50,8 @@ import com.google.inject.Provides;
public class CrowdsecExtension implements GuiceModuleTestExtension {
public static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(5);
public static final int CROWDSEC_PORT = 8080;
- public static final String CROWDSEC_IMAGE =
"crowdsecurity/crowdsec:v1.5.4";
+ public static final String CROWDSEC_IMAGE =
"crowdsecurity/crowdsec:v1.6.4";
+ private static final Logger LOGGER =
LoggerFactory.getLogger("tc.crowdsec");
private final GenericContainer<?> crowdsecContainer;
@@ -56,12 +60,15 @@ public class CrowdsecExtension implements
GuiceModuleTestExtension {
.withCreateContainerCmdModifier(cmd ->
cmd.withName("james-crowdsec-test-" + UUID.randomUUID()))
.withExposedPorts(CROWDSEC_PORT)
.withStartupTimeout(STARTUP_TIMEOUT)
+
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/acquis.yaml"),
"/etc/crowdsec/")
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/parsers/syslog-logs.yaml"),
"/etc/crowdsec/parsers/s00-raw/")
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/parsers/james-auth.yaml"),
"/etc/crowdsec/parsers/s01-parse/")
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/scenarios/james-bf-auth.yaml"),
"/etc/crowdsec/scenarios/")
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/scenarios/james-dictionary-attack.yaml"),
"/etc/crowdsec/scenarios/")
.withCopyFileToContainer(MountableFile.forClasspathResource("crowdsec/collections/james.yaml"),
"/etc/crowdsec/collections/")
+ .withEnv("DISABLE_PARSERS", "crowdsecurity/whitelists
crowdsecurity/geoip-enrich crowdsecurity/dateparse-enrich")
+ .withLogConsumer(new Slf4jLogConsumer(LOGGER))
.withFileSystemBind("src/test/resources/log", "/var/log",
BindMode.READ_WRITE)
.waitingFor(new
HostPortWaitStrategy().withRateLimiter(RateLimiters.TWENTIES_PER_SECOND));
}
diff --git
a/third-party/crowdsec/src/test/resources/crowdsec/parsers/syslog-logs.yaml
b/third-party/crowdsec/src/test/resources/crowdsec/parsers/syslog-logs.yaml
index 1b4f50f34e..a349f332af 100644
--- a/third-party/crowdsec/src/test/resources/crowdsec/parsers/syslog-logs.yaml
+++ b/third-party/crowdsec/src/test/resources/crowdsec/parsers/syslog-logs.yaml
@@ -64,11 +64,11 @@ statics:
- parsed: mdc_username
expression: evt.Unmarshaled.message.mdc.username
- parsed: logger
- expression: evt.Unmarshaled.message.logger
+ expression: evt.Unmarshaled.message.loggerName
- parsed: message
- expression: evt.Unmarshaled.message.message
+ expression: evt.Unmarshaled.message.formattedMessage
- parsed: context
- expression: evt.Unmarshaled.message.context
+ expression: ToJsonString(evt.Unmarshaled.message.context)
- meta: datasource_path
expression: evt.Line.Src
- meta: datasource_type
diff --git a/third-party/crowdsec/src/test/resources/logback-test.xml
b/third-party/crowdsec/src/test/resources/logback-test.xml
index ad3f309937..fd1c10bdca 100644
--- a/third-party/crowdsec/src/test/resources/logback-test.xml
+++ b/third-party/crowdsec/src/test/resources/logback-test.xml
@@ -37,23 +37,29 @@
<maxFileSize>100MB</maxFileSize>
</triggeringPolicy>
- <encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
- <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
- <timestampFormat>yyyy-MM-dd'T'HH:mm:ss.SSSX</timestampFormat>
- <timestampFormatTimezoneId>Etc/UTC</timestampFormatTimezoneId>
-
- <!-- Importance for handling multiple lines log -->
- <appendLineSeparator>true</appendLineSeparator>
-
- <jsonFormatter
class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter">
- <prettyPrint>false</prettyPrint>
- </jsonFormatter>
- </layout>
+ <encoder class="ch.qos.logback.classic.encoder.JsonEncoder">
+ <withFormattedMessage>true</withFormattedMessage>
</encoder>
+ <immediateFlush>false</immediateFlush>
</appender>
<root level="WARN">
<appender-ref ref="LOG_FILE" />
+<!-- <appender-ref ref="CONSOLE" />-->
</root>
<logger name="org.apache.james" level="INFO" />
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} -
%msg%n%rEx</pattern>
+ </encoder>
+ <immediateFlush>false</immediateFlush>
+ <!-- Disable below block for debug logs mode-->
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
+<!-- <logger name="tc" level="INFO" additivity="false">-->
+<!-- <appender-ref ref="CONSOLE"/>-->
+<!-- </logger>-->
</configuration>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscr...@james.apache.org
For additional commands, e-mail: notifications-h...@james.apache.org
