SENSSOFT-55 #Added Docker support.
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/commit/e8c71a1b Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/tree/e8c71a1b Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/diff/e8c71a1b Branch: refs/heads/master Commit: e8c71a1b0fe6f959b986a8cd016a3ca26b38c97e Parents: fb6c43a Author: mooshu1x2 <mbe...@draper.com> Authored: Mon Sep 19 17:10:45 2016 -0400 Committer: mooshu1x2 <mbe...@draper.com> Committed: Mon Sep 19 17:10:45 2016 -0400 ---------------------------------------------------------------------- .dockerignore | 26 ++++++ distill/config.cfg | 2 +- distill/deploy/run_server.py | 29 ------- distill/server.py | 29 +++++++ docker-compose.yml | 65 ++++++++++++++ docker/distill/Dockerfile | 35 ++++++++ docker/distill/distill.conf | 29 +++++++ docker/es/Dockerfile | 62 ++++++++++++++ docker/es/elasticsearch.yml | 94 ++++++++++++++++++++ docker/es/logging.yml | 85 ++++++++++++++++++ docker/kibana/Dockerfile | 60 +++++++++++++ docker/kibana/entrypoint.sh | 24 ++++++ docker/kibana/kibana.yml | 79 +++++++++++++++++ docker/logstash/Dockerfile | 46 ++++++++++ docker/logstash/logstash.conf | 69 +++++++++++++++ docker/logstash/userale.json | 152 +++++++++++++++++++++++++++++++++ docker/webserver/Dockerfile | 34 ++++++++ docker/webserver/gunicorn.cfg | 27 ++++++ docker/webserver/nginx.conf | 57 +++++++++++++ docker/webserver/supervisord.conf | 13 +++ setup.py | 2 +- 21 files changed, 988 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/.dockerignore ---------------------------------------------------------------------- diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e2dfc86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*.rst +*.in +LICENSE +NOTICE +env +userale +es +docs +doc_requirements.txt +docker +docker-compose.yml http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/config.cfg ---------------------------------------------------------------------- diff --git a/distill/config.cfg b/distill/config.cfg index b1fdfa7..189e3ea 100644 --- a/distill/config.cfg +++ b/distill/config.cfg @@ -30,7 +30,7 @@ MAPPINGS = '../path/to/mappings/MOT_Mappings.csv' SELECTED = '../path/to/stout/selected_vars_for_distill.csv' # Elasticsearch Configuration -ES_HOST = 'http://localhost' +ES_HOST = 'http://elasticsearch' ES_PORT = 9200 HTTP_AUTH = None USE_SSL = False http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/deploy/run_server.py ---------------------------------------------------------------------- diff --git a/distill/deploy/run_server.py b/distill/deploy/run_server.py deleted file mode 100644 index 23acd83..0000000 --- a/distill/deploy/run_server.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from distill import app -from distill.app import * - -""" -Start up a local WSGI server called development -""" -def dev_server (): - host = app.config ['HOST'] - port = app.config ['PORT'] - debug = app.config ['DEBUG'] - app.run (host=host, port=port, debug=debug) - -if __name__ == '__main__': - dev_server () http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/distill/server.py ---------------------------------------------------------------------- diff --git a/distill/server.py b/distill/server.py new file mode 100644 index 0000000..23acd83 --- /dev/null +++ b/distill/server.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distill import app +from distill.app import * + +""" +Start up a local WSGI server called development +""" +def dev_server (): + host = app.config ['HOST'] + port = app.config ['PORT'] + debug = app.config ['DEBUG'] + app.run (host=host, port=port, debug=debug) + +if __name__ == '__main__': + dev_server () http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d546c6f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache Distill Production Docker build + +version: "2" +services: + distill: + build: + context: . + dockerfile: ./docker/distill/Dockerfile + container_name: distill + ports: + - 8090:8090 + depends_on: + - elasticsearch + links: + - elasticsearch + # Web server + #webserver: + # build: ./docker/webserver + # container_name: webserver + # links: + # - distill + # ELK Stack + elasticsearch: + build: ./docker/es + container_name: elastic + ports: + - 9200:9200 + - 9300:9300 + volumes: + - ./es/data:/usr/share/elasticsearch/data + - ./es/logs:/usr/share/elasticsearch/logs + kibana: + build: ./docker/kibana + container_name: kibana + ports: + - 5601:5601 + depends_on: + - elasticsearch + links: + - elasticsearch + logstash: + build: ./docker/logstash + container_name: logstash + command: logstash -f /etc/logstash/conf.d/logstash.conf + volumes: + - ./userale:/var/log/sennsoft + depends_on: + - elasticsearch + links: + - elasticsearch http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/distill/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/distill/Dockerfile b/docker/distill/Dockerfile new file mode 100644 index 0000000..807ff3a --- /dev/null +++ b/docker/distill/Dockerfile @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2 + +# install system wide deps +RUN apt-get -yqq update + +# Add code +WORKDIR /app + +ADD ./requirements.txt /app + +# Install Distill dependencies +RUN pip install -r requirements.txt + +# Expose Ports +EXPOSE 8090 + +# Rest +ADD . /app +RUN python setup.py develop +CMD python distill/server.py \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/distill/distill.conf ---------------------------------------------------------------------- diff --git a/docker/distill/distill.conf b/docker/distill/distill.conf new file mode 100644 index 0000000..a113dcb --- /dev/null +++ b/docker/distill/distill.conf @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description "Gunicorn application server running Apache Distill" + +# Restart process if it ever fails +start on runlevel [2345] +stop on runlevel [!2345] + +respawn +#Setup user and group that Gunicorn should be run as +setuid nobody +setgid www-data + +# Path to run_server +chdir /path/to/distill +exec gunicorn -c "gunicorn.cfg" scripts/run_server:app http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile new file mode 100644 index 0000000..b84b637 --- /dev/null +++ b/docker/es/Dockerfile @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ubuntu Xenial 16.04 [LTS] +FROM ubuntu:16.04 + +# install system wide deps +RUN apt-get -yqq update +RUN apt-get -yqq install openjdk-8-jre + +RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 + +ENV ELASTICSEARCH_VERSION 2.3.5 +ENV ELASTICSEARCH_REPO_BASE http://packages.elasticsearch.org/elasticsearch/2.x/debian + +RUN echo "deb $ELASTICSEARCH_REPO_BASE stable main" > /etc/apt/sources.list.d/elasticsearch.list + +# install elasticsearch +RUN set -x \ + && apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends elasticsearch=$ELASTICSEARCH_VERSION \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH /usr/share/elasticsearch/bin:$PATH + +WORKDIR /usr/share/elasticsearch + +RUN set -ex \ + && for path in \ + ./data \ + ./logs \ + ./config \ + ./config/scripts \ + ; do \ + mkdir -p "$path"; \ + chown -R elasticsearch:elasticsearch "$path"; \ + done + +COPY elasticsearch.yml /usr/share/elasticsearch/config +COPY logging.yml /usr/share/elasticsearch/config + +USER elasticsearch + +# Install Elastic-HQ +RUN plugin install royrusso/elasticsearch-HQ/2.0.3 + +CMD ["elasticsearch"] + +# Run on ports 9200 & 9300 +EXPOSE 9200 9300 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml new file mode 100644 index 0000000..6126feb --- /dev/null +++ b/docker/es/elasticsearch.yml @@ -0,0 +1,94 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please see the documentation for further information on configuration options: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html> +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: SensSoft +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: soft-01 +# +# Add custom attributes to the node: +# +# node.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: /usr/share/elasticsearch/data +# +# Path to log files: +# +path.logs: /usr/share/elasticsearch/logs +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +# bootstrap.mlockall: true +# +# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory +# available on the system and that the owner of the process is allowed to use this limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +network.host: 0.0.0.0 +# +# Set a custom port for HTTP: +# +# http.port: 9200 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html> +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2"] +# +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +# +# discovery.zen.minimum_master_nodes: 3 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html> +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +# gateway.recover_after_nodes: 3 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html> +# +# ---------------------------------- Various ----------------------------------- +# +# Disable starting multiple nodes on a single system: +# +# node.max_local_storage_nodes: 1 +# +# Require explicit names when deleting indices: +# +# action.destructive_requires_name: true http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/es/logging.yml ---------------------------------------------------------------------- diff --git a/docker/es/logging.yml b/docker/es/logging.yml new file mode 100644 index 0000000..939aa1e --- /dev/null +++ b/docker/es/logging.yml @@ -0,0 +1,85 @@ +# you can override this using by setting a system property, for example -Des.logger.level=DEBUG +es.logger.level: INFO +rootLogger: ${es.logger.level}, console, file +logger: + # log action execution errors for easier debugging + action: DEBUG + + # deprecation logging, turn to DEBUG to see them + deprecation: INFO, deprecation_log_file + + # reduce the logging for aws, too much is logged under the default INFO + com.amazonaws: WARN + # aws will try to do some sketchy JMX stuff, but its not needed. + com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR + com.amazonaws.metrics.AwsSdkMetrics: ERROR + + org.apache.http: INFO + + # gateway + #gateway: DEBUG + #index.gateway: DEBUG + + # peer shard recovery + #indices.recovery: DEBUG + + # discovery + #discovery: TRACE + + index.search.slowlog: TRACE, index_search_slow_log_file + index.indexing.slowlog: TRACE, index_indexing_slow_log_file + +additivity: + index.search.slowlog: false + index.indexing.slowlog: false + deprecation: false + +appender: + console: + type: console + layout: + type: consolePattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %.10000m%n" + + # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files. + # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html + #file: + #type: extrasRollingFile + #file: ${path.logs}/${cluster.name}.log + #rollingPolicy: timeBased + #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz + #layout: + #type: pattern + #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + deprecation_log_file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}_deprecation.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + index_search_slow_log_file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}_index_search_slowlog.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" + + index_indexing_slow_log_file: + type: dailyRollingFile + file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log + datePattern: "'.'yyyy-MM-dd" + layout: + type: pattern + conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile new file mode 100644 index 0000000..e37cfa0 --- /dev/null +++ b/docker/kibana/Dockerfile @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ubuntu Xenial 16.04 [LTS] +FROM ubuntu:16.04 + +# install system wide deps +RUN apt-get -yqq update +RUN apt-get -yqq install netcat + +RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 + +ENV KIBANA_VERSION 4.5.4 +ENV KIBANA_REPO_BASE http://packages.elasticsearch.org/kibana/4.5/debian + +RUN echo "deb $KIBANA_REPO_BASE stable main" > /etc/apt/sources.list.d/kibana.list + +# install kibana +RUN set -x \ + && apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends kibana=$KIBANA_VERSION \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH /opt/kibana/bin:$PATH + +WORKDIR /opt/kibana + +RUN set -ex \ + && for path in \ + ./config \ + ; do \ + chown -R kibana:kibana "$path"; \ + done + +COPY kibana.yml /opt/kibana/config + +COPY entrypoint.sh /tmp/entrypoint.sh +RUN chmod +x /tmp/entrypoint.sh + +# Install Sense plugin +RUN kibana plugin --install elastic/sense + +USER kibana + +CMD ["/tmp/entrypoint.sh"] + +# Run on port 5601 +EXPOSE 5601 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/entrypoint.sh ---------------------------------------------------------------------- diff --git a/docker/kibana/entrypoint.sh b/docker/kibana/entrypoint.sh new file mode 100644 index 0000000..02bd91f --- /dev/null +++ b/docker/kibana/entrypoint.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Wait for the Elasticsearch container to be ready before starting Kibana. +echo "Stalling for Elasticsearch" +while true; do + nc -q 1 elasticsearch 9200 2>/dev/null && break +done + +echo "Starting Kibana" +exec kibana \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/kibana/kibana.yml ---------------------------------------------------------------------- diff --git a/docker/kibana/kibana.yml b/docker/kibana/kibana.yml new file mode 100644 index 0000000..8bc978f --- /dev/null +++ b/docker/kibana/kibana.yml @@ -0,0 +1,79 @@ +# Kibana is served by a back end server. This controls which port to use. +server.port: 5601 + +# The host to bind the server to. +server.host: "0.0.0.0" + +# If you are running kibana behind a proxy, and want to mount it at a path, +# specify that path here. The basePath can't end in a slash. +# server.basePath: "" + +# The maximum payload size in bytes on incoming server requests. +# server.maxPayloadBytes: 1048576 + +# The Elasticsearch instance to use for all your queries. +elasticsearch.url: "http://elasticsearch:9200" + +# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, +# then the host you use to connect to *this* Kibana instance will be sent. +elasticsearch.preserveHost: true + +# Kibana uses an index in Elasticsearch to store saved searches, visualizations +# and dashboards. It will create a new index if it doesn't already exist. +kibana.index: ".kibana" + +# The default application to load. +kibana.defaultAppId: "discover" + +# If your Elasticsearch is protected with basic auth, these are the user credentials +# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana +# users will still need to authenticate with Elasticsearch (which is proxied through +# the Kibana server) +# elasticsearch.username: "user" +# elasticsearch.password: "pass" + +# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) +# server.ssl.cert: /path/to/your/server.crt +# server.ssl.key: /path/to/your/server.key + +# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) +# elasticsearch.ssl.cert: /path/to/your/client.crt +# elasticsearch.ssl.key: /path/to/your/client.key + +# If you need to provide a CA certificate for your Elasticsearch instance, put +# the path of the pem file here. +# elasticsearch.ssl.ca: /path/to/your/CA.pem + +# Set to false to have a complete disregard for the validity of the SSL +# certificate. +# elasticsearch.ssl.verify: true + +# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to +# request_timeout setting +# elasticsearch.pingTimeout: 1500 + +# Time in milliseconds to wait for responses from the back end or elasticsearch. +# This must be > 0 +# elasticsearch.requestTimeout: 30000 + +# Time in milliseconds for Elasticsearch to wait for responses from shards. +# Set to 0 to disable. +# elasticsearch.shardTimeout: 0 + +# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying +# elasticsearch.startupTimeout: 5000 + +# Set the path to where you would like the process id file to be created. +# pid.file: /var/run/kibana.pid + +# If you would like to send the log output to a file you can set the path below. +logging.dest: stdout + +# Set this to true to suppress all logging output. +# logging.silent: false + +# Set this to true to suppress all logging output except for error messages. +logging.quiet: true + +# Set this to true to log all events, including system usage information and all requests. +# logging.verbose: false http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile new file mode 100644 index 0000000..bd5929b --- /dev/null +++ b/docker/logstash/Dockerfile @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Ubuntu Xenial 16.04 [LTS] +FROM ubuntu:16.04 + +# install system wide deps +RUN apt-get -yqq update +RUN apt-get -yqq install openjdk-8-jre +RUN apt-get -yqq install wget +RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 46095ACC8548582C1A2699A9D27D666CD88E42B4 + +ENV LOGSTASH_VERSION 1:2.3.4-1 +ENV LOGSTASH_REPO_BASE http://packages.elasticsearch.org/logstash/2.3/debian + +RUN echo "deb $LOGSTASH_REPO_BASE stable main" > /etc/apt/sources.list.d/logstash.list + +# install logstash +RUN set -x \ + && apt-get -yqq update \ + && apt-get -yqq install --no-install-recommends logstash=$LOGSTASH_VERSION \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH /opt/logstash/bin:$PATH + +# Get GeoIP DB +#RUN set -x \ +# && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \ +# && gunzip GeoIP.dat.gz \ +# && mkdir -p /usr/share/GeoIP \ +# && mv GeoIP.dat /usr/share/GeoIP + +ADD logstash.conf /etc/logstash/conf.d/logstash.conf +ADD userale.json /etc/logstash/conf.d/userale.json \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/logstash.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/logstash.conf b/docker/logstash/logstash.conf new file mode 100644 index 0000000..09b5caf --- /dev/null +++ b/docker/logstash/logstash.conf @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +input { + file { + path => [ + "/var/log/sennsoft/*.log" + ] + codec => "json" + start_position => "beginning" + sincedb_path => "/dev/null" + } +} + +## Add your filters / logstash plugins configuration here +filter { + mutate { + remove_field => [ "message", "@version", "@timestamp"] + } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } + + #dns { + # add_field => [ "hostname", "%{host}" ] + #} + + #dns { + # resolve => [ "host" ] + # action => [ "replace" ] + #} + + #geoip { + # source => "host" + # database => "/usr/share/GeoIP/GeoIP.dat" + # target => "geoip" + #} +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "elasticsearch:9200" + action => "index" + index => "userale" + document_type => "logs" + manage_template => true + template_overwrite => true + template => "/etc/logstash/conf.d/userale.json" + template_name => "userale" + } + + # Debug + stdout { codec => rubydebug } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/logstash/userale.json ---------------------------------------------------------------------- diff --git a/docker/logstash/userale.json b/docker/logstash/userale.json new file mode 100644 index 0000000..3d53d7b --- /dev/null +++ b/docker/logstash/userale.json @@ -0,0 +1,152 @@ +{ + "template" : "userale", + "order" : 1, + "settings" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "index.refresh_interval" : "5s" + }, + "mappings" : { + "_default_" : { + "_all" : { + "enabled" : false + }, + "properties" : { + "geoip" : { + "properties" : { + "ip" : { + "type" : "ip" + }, + "country_code2" : { + "index" : "not_analyzed", + "type" : "string" + }, + "country_code3" : { + "index" : "not_analyzed", + "type" : "string" + }, + "country_name" : { + "index" : "not_analyzed", + "type" : "string" + }, + "continent_code" : { + "index" : "not_analyzed", + "type" : "string" + }, + "region_name" : { + "index" : "not_analyzed", + "type" : "string" + }, + "city_name" : { + "index" : "not_analyzed", + "type" : "string" + }, + "postal_code" : { + "index" : "not_analyzed", + "type" : "string" + }, + "latitude" : { + "type" : "double" + }, + "longitude" : { + "type" : "double" + }, + "dma_code" : { + "type" : "integer" + }, + "area_code" : { + "type" : "long" + }, + "timezone" : { + "index" : "not_analyzed", + "type" : "string" + }, + "real_region_name" : { + "index" : "not_analyzed", + "type" : "string" + }, + "location" : { + "doc_values" : true, + "type" : "geo_point" + } + } + } + } + }, + "logs" : { + "properties" : { + "userAction" : { + "type" : "boolean" + }, + "type" : { + "index" : "not_analyzed", + "type" : "string" + }, + "clientTime" : { + "type" : "date", + "format" : "strict_date_optional_time||epoch_millis" + }, + "target" : { + "index" : "not_analyzed", + "type" : "string" + }, + "path" : { + "index" : "not_analyzed", + "type" : "string" + }, + "details" : { + "type" : "object" + }, + "location" : { + "type" : "object", + "properties" : { + "x" : { + "type" : "integer" + }, + "y" : { + "type" : "integer" + } + } + }, + "userId" : { + "index" : "not_analyzed", + "type" : "string" + }, + "session" : { + "index" : "not_analyzed", + "type" : "string" + }, + "toolName" : { + "index" : "not_analyzed", + "type" : "string" + }, + "toolVersion" : { + "index" : "not_analyzed", + "type" : "string" + }, + "useraleVersion" : { + "index" : "not_analyzed", + "type" : "string" + }, + "major_ver" : { + "type" : "integer" + }, + "minor_ver" : { + "type" : "integer" + }, + "patch_ver" : { + "type" : "integer" + }, + "tool_major_ver" : { + "type" : "integer" + }, + "tool_minor_ver" : { + "type" : "integer" + }, + "tool_patch_ver" : { + "type" : "integer" + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/webserver/Dockerfile b/docker/webserver/Dockerfile new file mode 100644 index 0000000..99fc7a5 --- /dev/null +++ b/docker/webserver/Dockerfile @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2 + +# install system wide deps +RUN apt-get -yqq update +RUN apt-get -yqq install nginx + +# Install Supervisord +RUN pip install supervisor-stdout + +# Add configuration files +ADD ./supervisord.conf /etc/supervisord.conf +ADD ./gunicorn.cfg /etc/gunicorn.cfg +ADD ./nginx.conf /etc/nginx/nginx.conf + +# restart nginx to load the config +RUN service nginx stop + +# start supervisor to run our wsgi server +CMD supervisord -c /etc/supervisord.conf -n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/gunicorn.cfg ---------------------------------------------------------------------- diff --git a/docker/webserver/gunicorn.cfg b/docker/webserver/gunicorn.cfg new file mode 100644 index 0000000..5b199c4 --- /dev/null +++ b/docker/webserver/gunicorn.cfg @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +gunicorn WSGI server configuration. +""" + +import multiprocessing + +bind = ["unix:distill.sock"] +workers = max (multiprocessing.cpu_count () * 2 + 1, 4) +timeout = 10 +umask = 007 +accesslog = "-" +errorlog = "-" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/nginx.conf ---------------------------------------------------------------------- diff --git a/docker/webserver/nginx.conf b/docker/webserver/nginx.conf new file mode 100644 index 0000000..3a46ed8 --- /dev/null +++ b/docker/webserver/nginx.conf @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +sudo /etc/init.d/nginx start +sudo rm /etc/nginx/sites-enabled/default +sudo touch /etc/nginx/sites-available/distill +sudo ln -s /etc/nginx/sites-available/distill /etc/nginx/sites-enabled/distill + +sudo vim /etc/nginx/sites-enabled/distill + +user nobody; +worker_processes 2; + +error_log /var/log/nginx/error.log; +error_log /var/log/nginx/error.log notice; +error_log /var/log/nginx/error.log info; + +# user nobody nogroup +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + access_log /var/log/nginx/access.log combined; + sendfile on; + + server { + listen 80 default; + server_name localhost; + keepalive_timeout 5; + + access_log /var/log/nginx/distill.log; + + location / { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $host; + proxy_pass http://127.0.0.1:8000; + } + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/docker/webserver/supervisord.conf ---------------------------------------------------------------------- diff --git a/docker/webserver/supervisord.conf b/docker/webserver/supervisord.conf new file mode 100644 index 0000000..bb15542 --- /dev/null +++ b/docker/webserver/supervisord.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon = true + +[program:nginx] +command = /usr/sbin/nginx +startsecs = 5 +stdout_events_enabled = true +stderr_events_enabled = true + +[program:app-gunicorn] +command = gunicorn -c "/etc/gunicorn.cfg" scripts/run_server:app +stdout_events_enabled = true +stderr_events_enabled = true http://git-wip-us.apache.org/repos/asf/incubator-senssoft-distill/blob/e8c71a1b/setup.py ---------------------------------------------------------------------- diff --git a/setup.py b/setup.py index ccf0855..aa474d2 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ setup ( ], entry_points = { 'console_scripts': [ - 'dev = distill.deploy.run_server:dev_server' + 'dev = distill.server:dev_server' ] } )