This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit c35688a3bef89037c799540a4ca6854ccac9d389 Author: Oliver Szabo <[email protected]> AuthorDate: Tue Oct 2 01:34:24 2018 +0200 Add Log Searach container support --- Makefile | 38 +++++++++++++ ambari-logsearch-it/pom.xml | 4 +- ambari-logsearch-server/build.xml | 6 ++ ambari-logsearch-server/pom.xml | 19 +++++++ .../containers/docker-logsearch-portal/Dockerfile | 49 ++++++++++++++++ .../docker-logsearch-portal/bin/entrypoint.sh | 32 +++++++++++ .../containers/docker-logsearch-portal/bin/init.sh | 13 +++++ .../docker-logsearch-portal/conf/log4j2.yml | 50 ++++++++++++++++ .../docker-logsearch-portal/conf/logsearch-env.sh | 13 +++++ .../conf/logsearch.properties | 66 ++++++++++++++++++++++ pom.xml | 7 +++ 11 files changed, 295 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a23e441 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GIT_REV_SHORT = $(shell git rev-parse --short HEAD) +MAVEN_BINARY ?= mvn + +ifeq ("$(USE_GIT_BUILD_TAG)", "true") + LOGSEARCH_BUILD_DOCKER_TAG = build-$(GIT_REV_SHORT) +else + LOGSEARCH_BUILD_DOCKER_TAG = "latest" +endif + +package: + $(MAVEN_BINARY) clean package + +test: + $(MAVEN_BINARY) clean test + +rpm: + $(MAVEN_BINARY) clean package -Dbuild-rpm -DskipTests + +deb: + $(MAVEN_BINARY) clean package -Dbuild-deb -DskipTests + +docker-build: + $(MAVEN_BINARY) clean package docker:build -DskipTests -Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG) + +docker-push: + $(MAVEN_BINARY) clean package docker:build docker:push -DskipTests -Dlogsearch.docker.tag=$(LOGSEARCH_BUILD_DOCKER_TAG) \ No newline at end of file diff --git a/ambari-logsearch-it/pom.xml b/ambari-logsearch-it/pom.xml index cf8970e..a4e160f 100644 --- a/ambari-logsearch-it/pom.xml +++ b/ambari-logsearch-it/pom.xml @@ -38,7 +38,7 @@ <jackson-jaxrs.version>2.9.4</jackson-jaxrs.version> <failsafe-plugin.version>2.20</failsafe-plugin.version> <forkCount>1</forkCount> - <docker.host>localhost</docker.host> + <logsearch.docker.host>localhost</logsearch.docker.host> <backend.stories.location>NONE</backend.stories.location> <ui.stories.location>NONE</ui.stories.location> </properties> @@ -283,7 +283,7 @@ </includes> <systemPropertyVariables> <log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration> - <docker.host>${docker.host}</docker.host> + <docker.host>${logsearch.docker.host}</docker.host> <backend.stories.location>${backend.stories.location}</backend.stories.location> </systemPropertyVariables> </configuration> diff --git a/ambari-logsearch-server/build.xml b/ambari-logsearch-server/build.xml index b509a15..88ccd6b 100644 --- a/ambari-logsearch-server/build.xml +++ b/ambari-logsearch-server/build.xml @@ -66,5 +66,11 @@ <exclude name="**/*.sh"/> </tarfileset> </tar> + <copy todir="target" > + <fileset dir="../jenkins/containers" includes="**"/> + </copy> + <copy todir="target/docker-logsearch-portal/logsearch-portal" > + <fileset dir="target/package" includes="**"/> + </copy> </target> </project> diff --git a/ambari-logsearch-server/pom.xml b/ambari-logsearch-server/pom.xml index 39a174e..5649361 100755 --- a/ambari-logsearch-server/pom.xml +++ b/ambari-logsearch-server/pom.xml @@ -41,6 +41,25 @@ <jjwt.version>0.6.0</jjwt.version> <javax-servlet.version>4.0.1</javax-servlet.version> </properties> + <build> + <plugins> + <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <configuration> + <verbose>true</verbose> + <images> + <image> + <name>apache/ambari-logsearch-portal:${logsearch.docker.tag}</name> + <build> + <dockerFileDir>${project.build.directory}/docker-logsearch-portal</dockerFileDir> + </build> + </image> + </images> + </configuration> + </plugin> + </plugins> + </build> <profiles> <profile> <id>dev</id> diff --git a/jenkins/containers/docker-logsearch-portal/Dockerfile b/jenkins/containers/docker-logsearch-portal/Dockerfile new file mode 100644 index 0000000..03ded63 --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/Dockerfile @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM openjdk:11-jre +MAINTAINER [email protected] + +ADD bin/entrypoint.sh /entrypoint.sh +ADD bin/init.sh /init.sh +ADD logsearch-portal /usr/lib/ambari-logsearch-portal + +ADD conf/log4j2.yml /usr/lib/ambari-logsearch-portal/conf/log4j2.yml +ADD conf/logsearch.properties /usr/lib/ambari-logsearch-portal/conf/logsearch.properties +ADD conf/logsearch-env.sh /usr/lib/ambari-logsearch-portal/conf/logsearch-env.sh + +ENV JAVA_HOME="/usr/java/default" +ENV LOGSEARCH_USER="logsearch" +ENV LOGSEARCH_GROUP="logsearch" +ENV LOGSEARCH_UID="61888" +ENV LOGSEARCH_GID="61888" +ENV LOGSEARCH_INIT_FILE /init.sh +ENV LOGSEARCH_PORT="61888" +ENV LOGSEARCH_PATH="/usr/lib/ambari-logsearch-portal" +ENV LOGSEARCH_CONF_DIR="/usr/lib/ambari-logsearch-portal/conf" +ENV LOGFILE="/var/log/ambari-logsearch-portal/logsearch.log" +ENV PID_FILE="/var/run/ambari-logsearch-portal/logsearch.pid" + +ENV LOGSEARCH_DEBUG=false +ENV LOGSEARCH_DEBUG_PORT=5005 +ENV LOGSEARCH_SSL="false" + +RUN groupadd -r --gid $LOGSEARCH_GID $LOGSEARCH_GROUP && useradd -r --uid $LOGSEARCH_UID --gid $LOGSEARCH_GID $LOGSEARCH_USER +RUN chown -R $LOGSEARCH_USER:$LOGSEARCH_GROUP /usr/lib/ambari-logsearch-portal +RUN find /usr/lib/ambari-logsearch-portal -type d -exec chmod 755 {} + +RUN find /usr/lib/ambari-logsearch-portal -type f -exec chmod 644 {} + +RUN chmod 755 $LOGSEARCH_PATH/bin/logsearch.sh +RUN chmod 755 $LOGSEARCH_PATH/conf/logsearch-env.sh +RUN chmod 755 /entrypoint.sh +RUN chmod 755 /init.sh + +WORKDIR /usr/lib/ambari-logsearch-portal +CMD ["/entrypoint.sh"] \ No newline at end of file diff --git a/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh b/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh new file mode 100644 index 0000000..c8baba4 --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/bin/entrypoint.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ZK_CONNECT_STRING=${ZK_CONNECT_STRING:-"localhost:9983"} + +function set_custom_zookeeper_address() { + local file_to_update=${1:?"usage: <filename_to_update>"} + local zk_connect_string="$ZK_CONNECT_STRING" + if [ "$zk_connect_string" != "localhost:9983" ] ; then + sed -i "s|localhost:9983|$zk_connect_string|g" $file_to_update + fi +} + +function start() { + set_custom_zookeeper_address /usr/lib/ambari-logsearch-portal/conf/logsearch.properties + /usr/lib/ambari-logsearch-portal/bin/logsearch.sh start -f +} + +if [[ -f "$LOGSEARCH_INIT_FILE" ]]; then + $LOGSEARCH_INIT_FILE +fi + +start ${@} diff --git a/jenkins/containers/docker-logsearch-portal/bin/init.sh b/jenkins/containers/docker-logsearch-portal/bin/init.sh new file mode 100644 index 0000000..72ce056 --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/bin/init.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +echo "No init file specified." \ No newline at end of file diff --git a/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml b/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml new file mode 100644 index 0000000..3e4fa63 --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/conf/log4j2.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Configutation: + name: LogSearchConfig + packages: org.apache.ambari.logsearch.layout + + Properties: + Property: + name: log-path + value: "/var/log/ambari-logsearch-portal" + + Appenders: + + Console: + name: Console_Appender + target: SYSTEM_OUT + PatternLayout: + pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n" + + RollingFile: + - name: RollingFile_Appender + fileName: ${log-path}/logsearch.log + filePattern: "logsearch-app.log.%d{yyyy-MM-dd-hh-mm}.gz" + PatternLayout: + pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n" + Policies: + SizeBasedTriggeringPolicy: + size: 10 MB + DefaultRollOverStrategy: + max: 10 + + Loggers: + + Root: + level: info + AppenderRef: + - ref: Console_Appender + - ref: RollingFile_Appender \ No newline at end of file diff --git a/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh b/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh new file mode 100644 index 0000000..7e0290f --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/conf/logsearch-env.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +LOGSEARCH_JAVA_MEM=${LOGSEARCH_JAVA_MEM:-"-Xmx1024m"} \ No newline at end of file diff --git a/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties b/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties new file mode 100644 index 0000000..59d2077 --- /dev/null +++ b/jenkins/containers/docker-logsearch-portal/conf/logsearch.properties @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logsearch.solr.zk_connect_string=localhost:9983 +logsearch.config.zk_connect_string=localhost:9983 + +logsearch.config.api.enabled=false +logsearch.config.api.filter.zk.enabled=true + +# Service Logs +logsearch.solr.collection.service.logs=service_logs +logsearch.solr.service.logs.config.name=hadoop_logs + +logsearch.collection.service.logs.numshards=2 +logsearch.collection.service.logs.replication.factor=2 + +# Audit logs +logsearch.solr.audit.logs.zk_connect_string=localhost:9983 +logsearch.solr.collection.audit.logs=audit_logs +logsearch.solr.audit.logs.url= + +logsearch.audit.logs.split.interval.mins=15 +logsearch.collection.audit.logs.numshards=2 +logsearch.collection.audit.logs.replication.factor=2 + +logsearch.solr.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets +logsearch.solr.audit.logs.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets + +# History logs +logsearch.solr.collection.history=history +logsearch.solr.history.config.name=history +logsearch.collection.history.replication.factor=1 + +# Metrics +logsearch.solr.metrics.collector.hosts= +logsearch.solr.jmx.port=18886 + +# Logfeeder Settings + +logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO + +# logsearch-admin.json +logsearch.auth.file.enable=true +logsearch.login.credentials.file=user_pass.json + +logsearch.auth.ldap.enable=false +logsearch.auth.simple.enable=false +logsearch.auth.external_auth.enable=false + +logsearch.auth.redirect.forward=true + +logsearch.protocol=http + +logsearch.session.timeout=30 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0f8b0e9..d6ecde5 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ <skipSurefireTests>false</skipSurefireTests> <compiler.version>3.8.0</compiler.version> <ambari-metrics.version>2.7.0.0.0</ambari-metrics.version> + <logsearch.docker.tag>latest</logsearch.docker.tag> </properties> <licenses> @@ -148,6 +149,11 @@ <pluginManagement> <plugins> <plugin> + <groupId>io.fabric8</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>0.27.1</version> + </plugin> + <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> <version>1.4.1</version> @@ -339,6 +345,7 @@ <exclude>**/docker/knox/**</exclude> <exclude>**/node_modules/**</exclude> <exclude>**/dist/**</exclude> + <exclude>.repository/**</exclude> </excludes> </configuration> <executions>
