Repository: incubator-senssoft-distill Updated Branches: refs/heads/elk5.0-upgrade [created] 923a3052e
Updated ELK stack to 5.0 Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/commit/923a3052 Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/tree/923a3052 Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/diff/923a3052 Branch: refs/heads/elk5.0-upgrade Commit: 923a3052e37028b93b7b68d94274fd6f4b146c12 Parents: a22b46c Author: mooshu1x2 <mbe...@draper.com> Authored: Tue Dec 6 17:59:32 2016 -0500 Committer: mooshu1x2 <mbe...@draper.com> Committed: Tue Dec 6 17:59:32 2016 -0500 ---------------------------------------------------------------------- docker-compose.yml | 22 +++++----- docker/es/Dockerfile | 50 ++-------------------- docker/es/elasticsearch.yml | 3 ++ docker/es/logging.yml | 85 -------------------------------------- docker/kibana/Dockerfile | 43 +++---------------- docker/logstash/Dockerfile | 30 ++++---------- docker/logstash/logstash.conf | 2 +- docker/logstash/userale.json | 79 ++++++++++++++++++----------------- 8 files changed, 73 insertions(+), 241 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index 52746b0..773151d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,11 +21,9 @@ services: build: context: . dockerfile: ./docker/distill/Dockerfile - container_name: distill + container_name: senssoft-distill ports: - 8090:8090 - depends_on: - - elasticsearch links: - elasticsearch # Web server @@ -37,31 +35,31 @@ services: # ELK Stack elasticsearch: build: ./docker/es - container_name: elastic + container_name: senssoft-elastic ports: - 9200:9200 - 9300:9300 volumes: - ./es/data:/usr/share/elasticsearch/data - ./es/logs:/usr/share/elasticsearch/logs + - ./docker/es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" kibana: build: ./docker/kibana - container_name: kibana + container_name: senssoft-kibana ports: - 5601:5601 - depends_on: - - elasticsearch links: - elasticsearch logstash: build: ./docker/logstash - container_name: logstash + container_name: senssoft-logstash ports: - 8000:8000 - command: logstash -f /etc/logstash/conf.d/logstash.conf volumes: - - ./userale:/var/log/sennsoft - depends_on: - - elasticsearch + - ./userale:/var/log/senssoft links: - elasticsearch + environment: + LS_HEAP_SIZE: "2048m" http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile index b84b637..8acde7c 100644 --- a/docker/es/Dockerfile +++ b/docker/es/Dockerfile @@ -13,50 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Ubuntu Xenial 16.04 [LTS] -FROM ubuntu:16.04 +FROM elasticsearch:5 +MAINTAINER Michelle Beard <msbe...@apache.org> -# install system wide deps -RUN apt-get -yqq update -RUN apt-get -yqq install openjdk-8-jre - -RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 - -ENV ELASTICSEARCH_VERSION 2.3.5 -ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/2.x/debian - -RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list - -# install elasticsearch -RUN set -x \ - && apt-get -yqq update \ - && apt-get -yqq install --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH /usr/share/elasticsearch/bin:$PATH - -WORKDIR /usr/share/elasticsearch - -RUN set -ex \ - && for path in \ - ./data \ - ./logs \ - ./config \ - ./config/scripts \ - ; do \ - mkdir -p "$path"; \ - chown -R elasticsearch:elasticsearch "$path"; \ - done - -COPY elasticsearch.yml /usr/share/elasticsearch/config -COPY logging.yml /usr/share/elasticsearch/config - -USER elasticsearch - -# Install Elastic-HQ -RUN plugin install royrusso/elasticsearch-HQ/2.0.3 - -CMD ["elasticsearch"] - -# Run on ports 9200 & 9300 -EXPOSE 9200 9300 \ No newline at end of file +# Install XPack +# RUN elasticsearch-plugin install --batch x-pack \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml index 6126feb..bac0b1d 100644 --- a/docker/es/elasticsearch.yml +++ b/docker/es/elasticsearch.yml @@ -92,3 +92,6 @@ network.host: 0.0.0.0 # Require explicit names when deleting indices: # # action.destructive_requires_name: true + +http.cors.enabled : true +http.cors.allow-origin : "*" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/es/logging.yml ---------------------------------------------------------------------- diff --git a/docker/es/logging.yml b/docker/es/logging.yml deleted file mode 100644 index 939aa1e..0000000 --- a/docker/es/logging.yml +++ /dev/null @@ -1,85 +0,0 @@ -# you can override this using by setting a system property, for example -Des.logger.level=DEBUG -es.logger.level: INFO -rootLogger: ${es.logger.level}, console, file -logger: - # log action execution errors for easier debugging - action: DEBUG - - # deprecation logging, turn to DEBUG to see them - deprecation: INFO, deprecation_log_file - - # reduce the logging for aws, too much is logged under the default INFO - com.amazonaws: WARN - # aws will try to do some sketchy JMX stuff, but its not needed. - com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR - com.amazonaws.metrics.AwsSdkMetrics: ERROR - - org.apache.http: INFO - - # gateway - #gateway: DEBUG - #index.gateway: DEBUG - - # peer shard recovery - #indices.recovery: DEBUG - - # discovery - #discovery: TRACE - - index.search.slowlog: TRACE, index_search_slow_log_file - index.indexing.slowlog: TRACE, index_indexing_slow_log_file - -additivity: - index.search.slowlog: false - index.indexing.slowlog: false - deprecation: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n" - - # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files. - # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html - #file: - #type: extrasRollingFile - #file: ${path.logs}/${cluster.name}.log - #rollingPolicy: timeBased - #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz - #layout: - #type: pattern - #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - deprecation_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_deprecation.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_indexing_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/kibana/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile index e37cfa0..0febc73 100644 --- a/docker/kibana/Dockerfile +++ b/docker/kibana/Dockerfile @@ -13,48 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Ubuntu Xenial 16.04 [LTS] -FROM ubuntu:16.04 +FROM kibana:5 +MAINTAINER Michelle Beard <msbe...@apache.org> -# install system wide deps -RUN apt-get -yqq update -RUN apt-get -yqq install netcat - -RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 - -ENV KIBANA_VERSION 4.5.4 -ENV KIBANA_REPO_BASE http://packages.elasticsearch.org/kibana/4.5/debian - -RUN echo "deb $KIBANA_REPO_BASE stable main" > /etc/apt/sources.list.d/kibana.list - -# install kibana -RUN set -x \ - && apt-get -yqq update \ - && apt-get -yqq install --no-install-recommends kibana=$KIBANA_VERSION \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH /opt/kibana/bin:$PATH - -WORKDIR /opt/kibana - -RUN set -ex \ - && for path in \ - ./config \ - ; do \ - chown -R kibana:kibana "$path"; \ - done - -COPY kibana.yml /opt/kibana/config +RUN apt-get update && apt-get install -y netcat bzip2 COPY entrypoint.sh /tmp/entrypoint.sh RUN chmod +x /tmp/entrypoint.sh -# Install Sense plugin -RUN kibana plugin --install elastic/sense - -USER kibana +# Install XPack +# RUN kibana-plugin install x-pack CMD ["/tmp/entrypoint.sh"] - -# Run on port 5601 -EXPOSE 5601 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile index bd5929b..c38251b 100644 --- a/docker/logstash/Dockerfile +++ b/docker/logstash/Dockerfile @@ -13,27 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Ubuntu Xenial 16.04 [LTS] -FROM ubuntu:16.04 +FROM logstash:5 +MAINTAINER Michelle Beard <msbe...@apache.org> -# install system wide deps -RUN apt-get -yqq update -RUN apt-get -yqq install openjdk-8-jre -RUN apt-get -yqq install wget -RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 - -ENV LOGSTASH_VERSION 1:2.3.4-1 -ENV LOGSTASH_REPO_BASE http://packages.elasticsearch.org/logstash/2.3/debian - -RUN echo "deb $LOGSTASH_REPO_BASE stable main" > /etc/apt/sources.list.d/logstash.list - -# install logstash -RUN set -x \ - && apt-get -yqq update \ - && apt-get -yqq install --no-install-recommends logstash=$LOGSTASH_VERSION \ - && rm -rf /var/lib/apt/lists/* - -ENV PATH /opt/logstash/bin:$PATH # Get GeoIP DB #RUN set -x \ @@ -42,5 +24,9 @@ ENV PATH /opt/logstash/bin:$PATH # && mkdir -p /usr/share/GeoIP \ # && mv GeoIP.dat /usr/share/GeoIP -ADD logstash.conf /etc/logstash/conf.d/logstash.conf -ADD userale.json /etc/logstash/conf.d/userale.json \ No newline at end of file +# Setup Logstash +COPY logstash.conf /etc/logstash/conf.d/logstash.conf +COPY userale.json /etc/logstash/conf.d/userale.json + +# Run Logstash +CMD ["logstash", "-f", "/etc/logstash/conf.d/logstash.conf"] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/logstash.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/logstash.conf b/docker/logstash/logstash.conf index ca831f5..375f3b9 100644 --- a/docker/logstash/logstash.conf +++ b/docker/logstash/logstash.conf @@ -25,7 +25,7 @@ input { file { path => [ - "/var/log/sennsoft/*.log" + "/var/log/senssoft/*.log" ] codec => "json" start_position => "beginning" http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/923a3052/docker/logstash/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/userale.json b/docker/logstash/userale.json index 3d53d7b..e5d8f91 100644 --- a/docker/logstash/userale.json +++ b/docker/logstash/userale.json @@ -18,38 +18,38 @@ "type" : "ip" }, "country_code2" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "country_code3" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "country_name" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "continent_code" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "region_name" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "city_name" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "postal_code" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "latitude" : { - "type" : "double" + "type" : "float" }, "longitude" : { - "type" : "double" + "type" : "float" }, "dma_code" : { "type" : "integer" @@ -58,12 +58,12 @@ "type" : "long" }, "timezone" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "real_region_name" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "location" : { "doc_values" : true, @@ -79,23 +79,28 @@ "type" : "boolean" }, "type" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "clientTime" : { "type" : "date", "format" : "strict_date_optional_time||epoch_millis" }, "target" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "path" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "details" : { - "type" : "object" + "type" : "object", + "properties" : { + "source" : { + "type" : "keyword" + } + } }, "location" : { "type" : "object", @@ -109,24 +114,24 @@ } }, "userId" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "session" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "toolName" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "toolVersion" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "useraleVersion" : { - "index" : "not_analyzed", - "type" : "string" + + "type" : "keyword" }, "major_ver" : { "type" : "integer"