This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch cloudbreak in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
commit c3a750a73e9f34f99494dad581563c2ed4307a15 Author: Olivér Szabó <[email protected]> AuthorDate: Thu Oct 25 15:36:31 2018 +0200 AMBARI-24825. Log Feeder: Fix HDFS/S3 outputs (#13) --- .../apache/ambari/logfeeder/output/OutputFile.java | 2 +- .../ambari/logfeeder/output/OutputHDFSFile.java | 7 ++ .../ambari/logfeeder/output/OutputS3File.java | 17 ++- .../ambari/logfeeder/output/spool/LogSpooler.java | 5 +- .../src/main/scripts/logfeeder.sh | 2 +- .../src/main/scripts/logsearch.sh | 2 +- docker/cloud-docker-compose.yml | 129 +++++++++++++++++++++ .../logfeeder/shipper-conf/output.config.json | 38 ++++++ 8 files changed, 196 insertions(+), 6 deletions(-) diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java index 7dae1b8..7495444 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java @@ -98,7 +98,7 @@ public class OutputFile extends Output<LogFeederProps, InputFileMarker> { String outStr = null; CSVPrinter csvPrinter = null; try { - if (codec.equals("csv")) { + if ("csv".equals(codec)) { csvPrinter = new CSVPrinter(outWriter, CSVFormat.RFC4180); //TODO: } else { diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java index ed93aa4..93a2643 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java @@ -39,6 +39,7 @@ import java.io.File; import java.util.Date; import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import java.util.concurrent.ConcurrentLinkedQueue; /** @@ -114,6 +115,12 @@ public class OutputHDFSFile extends Output<LogFeederProps, InputFileMarker> impl } } + @Override + public void write(Map<String, Object> jsonObj, InputFileMarker inputMarker) throws Exception { + String block = LogFeederUtil.getGson().toJson(jsonObj); + write(block, inputMarker); + } + @Override public String getShortDescription() { diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java index 7d7e6af..a2f6b08 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java @@ -192,12 +192,22 @@ public class OutputS3File extends OutputFile implements RolloverCondition, Rollo */ @Override public void write(String block, InputFileMarker inputMarker) { + createLogSpoolerIfRequired(inputMarker); + logSpooler.add(block); + } + + @Override + public void write(Map<String, Object> jsonObj, InputFileMarker inputMarker) throws Exception { + String block = LogFeederUtil.getGson().toJson(jsonObj); + write(block, inputMarker); + } + + private void createLogSpoolerIfRequired(InputFileMarker inputMarker) { if (logSpooler == null) { if (inputMarker.getInput().getClass().isAssignableFrom(InputFile.class)) { InputFile input = (InputFile) inputMarker.getInput(); logSpooler = createSpooler(input.getFilePath()); s3Uploader = createUploader(input.getInputDescriptor().getType()); - logSpooler.add(block); } else { logger.error("Cannot write from non local file..."); } @@ -261,4 +271,9 @@ public class OutputS3File extends OutputFile implements RolloverCondition, Rollo public void handleRollover(File rolloverFile) { s3Uploader.addFileForUpload(rolloverFile.getAbsolutePath()); } + + @Override + public String getShortDescription() { + return "output:destination=s3,bucket=" + s3OutputConfiguration.getS3BucketName(); + } } diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java index 7d7d111..82a3f1b 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java @@ -45,8 +45,9 @@ import java.util.concurrent.atomic.AtomicBoolean; public class LogSpooler { private static final Logger logger = LogManager.getLogger(LogSpooler.class); - public static final long TIME_BASED_ROLLOVER_DISABLED_THRESHOLD = 0; - static final String fileDateFormat = "yyyy-MM-dd-HH-mm-ss"; + + private static final String fileDateFormat = "yyyy-MM-dd-HH-mm-ss"; + private static final long TIME_BASED_ROLLOVER_DISABLED_THRESHOLD = 0; private String spoolDirectory; private String sourceFileNamePrefix; diff --git a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh index e125768..eaf00e1 100755 --- a/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh +++ b/ambari-logsearch-logfeeder/src/main/scripts/logfeeder.sh @@ -168,7 +168,7 @@ function start() { LOGFEEDER_DEBUG_PORT=${LOGFEEDER_DEBUG_PORT:-"5006"} if [ "$LOGFEEDER_DEBUG" = "true" ]; then - LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=*:$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " fi if [ "$LOGFEEDER_SSL" = "true" ]; then diff --git a/ambari-logsearch-server/src/main/scripts/logsearch.sh b/ambari-logsearch-server/src/main/scripts/logsearch.sh index b941e25..1b64832 100755 --- a/ambari-logsearch-server/src/main/scripts/logsearch.sh +++ b/ambari-logsearch-server/src/main/scripts/logsearch.sh @@ -149,7 +149,7 @@ function start() { LOGSEARCH_DEBUG_PORT=${LOGSEARCH_DEBUG_PORT:-"5005"} if [ "$LOGSEARCH_DEBUG" = "true" ]; then - LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND " + LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=*:$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND " fi if [ "$LOGSEARCH_SSL" = "true" ]; then diff --git a/docker/cloud-docker-compose.yml b/docker/cloud-docker-compose.yml new file mode 100644 index 0000000..3a9ec05 --- /dev/null +++ b/docker/cloud-docker-compose.yml @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +version: '3.3' +services: + zookeeper: + image: zookeeper:${ZOOKEEPER_VERSION:-3.4.10} + restart: always + hostname: zookeeper + networks: + - logsearch-network + ports: + - 2181:2181 + environment: + ZOO_MY_ID: 1 + ZOO_SERVERS: server.1=zookeeper:2888:3888 + solr: + image: solr:${SOLR_VERSION:-7.5.0} + restart: always + hostname: solr + ports: + - "8983:8983" + networks: + - logsearch-network + env_file: + - Profile + entrypoint: + - docker-entrypoint.sh + - solr + - start + - "-f" + - "-c" + - "-z" + - ${ZOOKEEPER_CONNECTION_STRING} + logsearch: + image: ambari-logsearch:v1.0 + restart: always + hostname: logsearch.apache.org + labels: + logfeeder.log.type: "logsearch_server" + networks: + - logsearch-network + env_file: + - Profile + ports: + - 61888:61888 + - 4444:4444 + - 5005:5005 + environment: + COMPONENT: logsearch + COMPONENT_LOG: logsearch + ZK_CONNECT_STRING: ${ZOOKEEPER_CONNECTION_STRING} + DISPLAY: $DISPLAY_MAC + volumes: + - $AMBARI_LOCATION:/root/ambari + - $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs + - $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config + logfeeder: + image: ambari-logsearch:v1.0 + restart: always + hostname: logfeeder.apache.org + privileged: true + labels: + logfeeder.log.type: "logfeeder" + networks: + - logsearch-network + env_file: + - Profile + ports: + - 5006:5006 + environment: + COMPONENT: logfeeder + COMPONENT_LOG: logfeeder + ZK_CONNECT_STRING: ${ZOOKEEPER_CONNECTION_STRING} + volumes: + - $AMBARI_LOCATION:/root/ambari + - $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs + - $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config + - /var/run/docker.sock:/var/run/docker.sock + - /usr/local/bin/docker:/usr/local/bin/docker + - /var/lib/docker:/var/lib/docker + fakes3: + image: localstack/localstack + hostname: fakes3 + ports: + - "4569:4569" + environment: + - SERVICES=s3:4569 + - DEBUG=s3 + networks: + logsearch-network: + aliases: + - testbucket.fakes3 + env_file: + - Profile + namenode: + image: flokkr/hadoop-hdfs-namenode:${HADOOP_VERSION:-3.0.0} + hostname: namenode + ports: + - 9870:9870 + - 9000:9000 + env_file: + - Profile + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-hdfs/dfs/name" + networks: + - logsearch-network + datanode: + image: flokkr/hadoop-hdfs-datanode:${HADOOP_VERSION:-3.0.0} + links: + - namenode + env_file: + - Profile + networks: + - logsearch-network +networks: + logsearch-network: + driver: bridge diff --git a/docker/test-config/logfeeder/shipper-conf/output.config.json b/docker/test-config/logfeeder/shipper-conf/output.config.json index a85b4a4..62950d1 100644 --- a/docker/test-config/logfeeder/shipper-conf/output.config.json +++ b/docker/test-config/logfeeder/shipper-conf/output.config.json @@ -31,6 +31,44 @@ ] } } + }, + { + "comment": "S3 file output", + "is_enabled": "true", + "destination": "s3_file", + "type": "s3", + "s3_access_key" : "accessKey", + "s3_secret_key" : "secretKey", + "s3_bucket" : "docker-logsearch", + "s3_endpoint" : "http://fakes3:4569";, + "s3_log_dir" : "/tmp", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "s3" + ] + } + } + }, + { + "comment": "HDFS file output", + "is_enabled": "true", + "destination": "hdfs", + "type": "hdfs", + "file_name_prefix":"service-logs-", + "hdfs_out_dir": "/logfeeder/$HOST/service", + "hdfs_host": "namenode", + "hdfs_port": "9000", + "rollover_sec":"10", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "hdfs" + ] + } + } } ] }
