This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new ec71546 Add Log Feeder container support.
ec71546 is described below
commit ec7154640b9309439173c6526f3e3a7033e71313
Author: Oliver Szabo <[email protected]>
AuthorDate: Tue Oct 2 17:08:37 2018 +0200
Add Log Feeder container support.
---
Makefile | 11 ++++-
ambari-logsearch-logfeeder/build.xml | 6 +++
ambari-logsearch-logfeeder/pom.xml | 15 ++++++
.../docker-logsearch-logfeeder/Dockerfile | 56 ++++++++++++++++++++++
.../docker-logsearch-logfeeder/bin/entrypoint.sh | 33 +++++++++++++
.../docker-logsearch-logfeeder/bin/init.sh | 13 +++++
.../docker-logsearch-logfeeder/conf/log4j2.yml | 49 +++++++++++++++++++
.../conf/logfeeder-env.sh | 13 +++++
.../conf/logfeeder.properties | 33 +++++++++++++
.../shipper-conf/global.config.json | 10 ++++
.../shipper-conf/output.config.json | 36 ++++++++++++++
11 files changed, 274 insertions(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index a23e441..e101e3f 100644
--- a/Makefile
+++ b/Makefile
@@ -22,6 +22,15 @@ endif
package:
$(MAVEN_BINARY) clean package
+install:
+ $(MAVEN_BINARY) clean install -DskipTests
+
+be:
+ $(MAVEN_BINARY) clean package -Pbe
+
+fe:
+ $(MAVEN_BINARY) clean package -Pfe
+
test:
$(MAVEN_BINARY) clean test
@@ -35,4 +44,4 @@ docker-build:
$(MAVEN_BINARY) clean package docker:build -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
docker-push:
- $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
\ No newline at end of file
+ $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests
-Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG)
diff --git a/ambari-logsearch-logfeeder/build.xml
b/ambari-logsearch-logfeeder/build.xml
index 1fbb928..15fedbc 100644
--- a/ambari-logsearch-logfeeder/build.xml
+++ b/ambari-logsearch-logfeeder/build.xml
@@ -51,5 +51,11 @@
<exclude name="**/*.sh"/>
</tarfileset>
</tar>
+ <copy todir="target" >
+ <fileset dir="../jenkins/containers" includes="**"/>
+ </copy>
+ <copy todir="target/docker-logsearch-logfeeder/logfeeder" >
+ <fileset dir="target/package" includes="**"/>
+ </copy>
</target>
</project>
diff --git a/ambari-logsearch-logfeeder/pom.xml
b/ambari-logsearch-logfeeder/pom.xml
index 2582bb8..8b89ff6 100644
--- a/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch-logfeeder/pom.xml
@@ -303,6 +303,21 @@
<plugins>
<plugin>
+ <groupId>io.fabric8</groupId>
+ <artifactId>docker-maven-plugin</artifactId>
+ <configuration>
+ <verbose>true</verbose>
+ <images>
+ <image>
+
<name>apache/ambari-logsearch-logfeeder:${logsearch.docker.tag}</name>
+ <build>
+
<dockerFileDir>${project.build.directory}/docker-logsearch-logfeeder</dockerFileDir>
+ </build>
+ </image>
+ </images>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
diff --git a/jenkins/containers/docker-logsearch-logfeeder/Dockerfile
b/jenkins/containers/docker-logsearch-logfeeder/Dockerfile
new file mode 100644
index 0000000..ab959a8
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/Dockerfile
@@ -0,0 +1,56 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM openjdk:11-jre
+MAINTAINER [email protected]
+
+ADD bin/entrypoint.sh /entrypoint.sh
+ADD bin/init.sh /init.sh
+ADD logfeeder /usr/lib/ambari-logsearch-logfeeder
+
+ADD conf/log4j2.yml /usr/lib/ambari-logsearch-logfeeder/conf/log4j2.yml
+ADD conf/logfeeder.properties
/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder.properties
+ADD conf/logfeeder-env.sh
/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder-env.sh
+
+ADD shipper-conf /shipper-conf
+
+ENV JAVA_HOME="/usr/java/default"
+ENV LOGFEEDER_USER="logfeeder"
+ENV LOGFEEDER_GROUP="logfeeder"
+ENV LOGFEEDER_UID="5555"
+ENV LOGFEEDER_GID="5555"
+ENV LOGSEARCH_INIT_FILE /init.sh
+ENV LOGFEEDER_PATH="/usr/lib/ambari-logsearch-logfeeder"
+ENV LOGFEEDER_CONF_DIR="/usr/lib/ambari-logsearch-logfeeder/conf"
+ENV LOG_PATH="/var/log/ambari-logsearch-logfeeder"
+ENV LOGFILE="/var/log/ambari-logsearch-logfeeder/logfeeder.log"
+ENV LOGFEEDER_PID_FILE="/var/run/ambari-logsearch-logfeeder/logfeeder.pid"
+
+ENV LOGFEEDER_DEBUG=false
+ENV LOGFEEDER_DEBUG_PORT=5005
+ENV LOGFEEDER_SSL="false"
+
+RUN groupadd -r --gid $LOGFEEDER_UID $LOGFEEDER_GROUP && useradd -r --uid
$LOGFEEDER_UID --gid $LOGFEEDER_GID $LOGFEEDER_USER
+RUN chown -R $LOGSEARCH_USER:$LOGSEARCH_GROUP
/usr/lib/ambari-logsearch-logfeeder
+RUN chown -R $LOGSEARCH_USER:$LOGSEARCH_GROUP /shipper-conf
+RUN mkdir /checkpoints && chown -R $LOGSEARCH_USER:$LOGSEARCH_GROUP
/checkpoints
+RUN mkdir -p /var/log/ambari-logsearch-logfeeder && chown -R
$LOGFEEDER_USER:$LOGFEEDER_GROUP /var/log/ambari-logsearch-logfeeder
+RUN mkdir -p /var/run/ambari-logsearch-logfeeder && chown -R
$LOGFEEDER_USER:$LOGFEEDER_GROUP /var/run/ambari-logsearch-logfeeder
+RUN find /usr/lib/ambari-logsearch-logfeeder -type d -exec chmod 755 {} +
+RUN find /usr/lib/ambari-logsearch-logfeeder -type f -exec chmod 644 {} +
+RUN chmod 755 $LOGFEEDER_PATH/bin/logfeeder.sh
+RUN chmod 755 $LOGFEEDER_PATH/conf/logfeeder-env.sh
+RUN chmod 755 /entrypoint.sh
+RUN chmod 755 /init.sh
+
+USER $LOGFEEDER_USER
+WORKDIR /usr/lib/ambari-logsearch-logfeeder
+CMD ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-logfeeder/bin/entrypoint.sh
b/jenkins/containers/docker-logsearch-logfeeder/bin/entrypoint.sh
new file mode 100644
index 0000000..b3d1846
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/bin/entrypoint.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ZK_CONNECT_STRING=${ZK_CONNECT_STRING:-"localhost:9983"}
+
+function set_custom_zookeeper_address() {
+ local file_to_update=${1:?"usage: <filename_to_update>"}
+ local zk_connect_string="$ZK_CONNECT_STRING"
+ if [ "$zk_connect_string" != "localhost:9983" ] ; then
+ sed -i "s|localhost:9983|$zk_connect_string|g" $file_to_update
+ fi
+}
+
+function start() {
+ set_custom_zookeeper_address
/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder.properties
+ set_custom_zookeeper_address /shipper-conf/output.config.json
+ /usr/lib/ambari-logsearch-logfeeder/bin/logfeeder.sh start -f
+}
+
+if [[ -f "$LOGFEEDER_INIT_FILE" ]]; then
+ $LOGFEEDER_INIT_FILE
+fi
+
+start ${@}
diff --git a/jenkins/containers/docker-logsearch-logfeeder/bin/init.sh
b/jenkins/containers/docker-logsearch-logfeeder/bin/init.sh
new file mode 100644
index 0000000..72ce056
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/bin/init.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+echo "No init file specified."
\ No newline at end of file
diff --git a/jenkins/containers/docker-logsearch-logfeeder/conf/log4j2.yml
b/jenkins/containers/docker-logsearch-logfeeder/conf/log4j2.yml
new file mode 100644
index 0000000..298b297
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/conf/log4j2.yml
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+Configutation:
+ name: LogFeederConfig
+ packages: org.apache.ambari.logsearch.layout
+
+ Properties:
+ Property:
+ name: log-path
+ value: "/var/log/ambari-logsearch-logfeeder"
+
+ Appenders:
+
+ Console:
+ name: Console_Appender
+ target: SYSTEM_OUT
+ PatternLayout:
+ pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n"
+
+ RollingFile:
+ - name: RollingFile_Appender
+ fileName: ${log-path}/logfeeder.log
+ filePattern: "logfeeder.log.%d{yyyy-MM-dd-hh-mm}.gz"
+ PatternLayout:
+ pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n"
+ Policies:
+ SizeBasedTriggeringPolicy:
+ size: 10 MB
+ DefaultRollOverStrategy:
+ max: 10
+
+ Loggers:
+
+ Root:
+ level: info
+ AppenderRef:
+ - ref: Console_Appender
\ No newline at end of file
diff --git
a/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder-env.sh
b/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder-env.sh
new file mode 100644
index 0000000..7ea8e65
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder-env.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+LOGFEEDER_JAVA_MEM=${LOGFEEDER_JAVA_MEM:-"-Xmx1024m"}
\ No newline at end of file
diff --git
a/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder.properties
b/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder.properties
new file mode 100644
index 0000000..548be99
--- /dev/null
+++ b/jenkins/containers/docker-logsearch-logfeeder/conf/logfeeder.properties
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cluster.name=CL1
+logfeeder.checkpoint.folder=/checkpoints
+logfeeder.metrics.collector.hosts=
+logfeeder.config.dir=/shipper-conf
+logfeeder.config.files=/shipper-conf/global.config.json,\
+ /shipper-conf/output.config.json
+logfeeder.log.filter.enable=true
+logfeeder.solr.config.interval=5
+logfeeder.solr.zk_connect_string=localhost:9983
+logfeeder.cache.enabled=true
+logfeeder.cache.size=100
+logfeeder.cache.key.field=log_message
+logfeeder.cache.dedup.interval=1000
+logfeeder.cache.last.dedup.enabled=true
+logsearch.config.zk_connect_string=localhost:9983
+logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
+logfeeder.docker.registry.enabled=true
+logfeeder.solr.core.config.name=history
\ No newline at end of file
diff --git
a/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/global.config.json
b/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/global.config.json
new file mode 100644
index 0000000..65da726
--- /dev/null
+++
b/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/global.config.json
@@ -0,0 +1,10 @@
+{
+ "global":{
+ "add_fields":{
+ "cluster":"cl1"
+ },
+ "source":"file",
+ "tail":"true",
+ "gen_event_md5":"true"
+ }
+}
diff --git
a/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/output.config.json
b/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/output.config.json
new file mode 100644
index 0000000..0ad1e01
--- /dev/null
+++
b/jenkins/containers/docker-logsearch-logfeeder/shipper-conf/output.config.json
@@ -0,0 +1,36 @@
+{
+ "output": [
+ {
+ "is_enabled": "true",
+ "comment": "Output to solr for service logs",
+ "collection" : "service_logs",
+ "destination": "solr",
+ "zk_connect_string": "localhost:9983",
+ "type": "service",
+ "skip_logtime": "true",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "service"
+ ]
+ }
+ }
+ },
+ {
+ "comment": "Output to solr for audit records",
+ "is_enabled": "true",
+ "collection" : "audit_logs",
+ "destination": "solr",
+ "zk_connect_string": "localhost:9983",
+ "type": "audit",
+ "skip_logtime": "true",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "audit"
+ ]
+ }
+ }
+ }
+ ]
+}
