Updated ELK setup
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/f30bf2af Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/f30bf2af Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/f30bf2af Branch: refs/heads/asf-site Commit: f30bf2af22190ce7edd969aa8d33432ff6fc81a8 Parents: b999347 Author: msb3399 <mbe...@draper.com> Authored: Wed Jul 12 23:31:19 2017 -0400 Committer: msb3399 <mbe...@draper.com> Committed: Wed Jul 12 23:31:19 2017 -0400 ---------------------------------------------------------------------- docker/docker-compose.yml | 131 +++++++++++++++------- docker/logstash/Dockerfile | 9 +- docker/logstash/config/logstash-userale.conf | 4 +- docker/startup/entrypoint.sh | 3 +- 4 files changed, 97 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/f30bf2af/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d0d0357..9dc69b7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,8 +23,8 @@ services: build: ./startup container_name: senssoft-start depends_on: - - "elasticsearch" - - "userale-forwarder" + - "elasticsearch1" + - "elasticsearch2" command: bash /opt/entrypoint.sh # Site @@ -33,33 +33,72 @@ services: container_name: senssoft-app ports: - 8080:8080 + depends_on: + - "userale-forwarder" command: python -m SimpleHTTPServer 8080 + networks: + - esnet # ELK Stack - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:5.4.0 - container_name: senssoft-elastic + + # The node elasticsearch1 listens on localhost:9200 while elasticsearch2 + # talks to elasticsearch1 over a Docker network. + elasticsearch1: + image: docker.elastic.co/elasticsearch/elasticsearch:5.5.0 + container_name: senssoft-elastic-1 + environment: + - cluster.name=SensSoft + - node.name=soft-01 + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g + volumes: + - esdata1:/usr/share/elasticsearch/data ports: - 9200:9200 - - 9300:9300 - volumes: - - ./es/data:/usr/share/elasticsearch/data - - ./es/logs:/usr/share/elasticsearch/logs - - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + networks: + - esnet + + elasticsearch2: + image: docker.elastic.co/elasticsearch/elasticsearch:5.5.0 + container_name: senssoft-elastic-2 environment: - ES_JAVA_OPTS: "-Xms4g -Xmx4g" + - cluster.name=SensSoft + - node.name=soft-02 + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "discovery.zen.ping.unicast.hosts=elasticsearch1" + ulimits: + memlock: + soft: -1 + hard: -1 + mem_limit: 1g + volumes: + - esdata2:/usr/share/elasticsearch/data + networks: + - esnet # Kibana kibana: - image: docker.elastic.co/kibana/kibana:5.4.0 + image: docker.elastic.co/kibana/kibana:5.5.0 container_name: senssoft-kibana ports: - 5601:5601 depends_on: - - "elasticsearch" + - "elasticsearch1" + - "elasticsearch2" - "startup" environment: - XPACK_SECURITY_ENABLED: "false" + XPACK_MONITORING_ENABLED: "false" + ELASTICSEARCH_URL: http://elasticsearch1:9200 + networks: + - esnet # Logstash userale-forwarder: @@ -72,34 +111,46 @@ services: - ./logstash/config/logstash-userale.conf:/etc/logstash/conf.d/logstash-userale.conf - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json depends_on: - - "elasticsearch" + - "elasticsearch1" + - "elasticsearch2" environment: - LS_HEAP_SIZE: "2048m" XPACK_MONITORING_ENABLED: "false" + networks: + - esnet - distill: - build: ./distill - container_name: distill - ports: - - 8090:8090 - depends_on: - - elasticsearch - links: - - elasticsearch +volumes: + esdata1: + driver: local + esdata2: + driver: local - db: - container_name: tap-db - build: ./db - ports: - - "5432:5432" +networks: + esnet: - tap: - container_name: tap-web - build: ./tap - command: ["bash", "./wait-for-postgres.sh"] - ports: - - "8000:8000" - depends_on: - - db - links: - - db:db \ No newline at end of file + + # distill: + # build: ./distill + # container_name: distill + # ports: + # - 8090:8090 + # depends_on: + # - elasticsearch + # links: + # - elasticsearch + + # db: + # container_name: tap-db + # build: ./db + # ports: + # - "5432:5432" + + # tap: + # container_name: tap-web + # build: ./tap + # command: ["bash", "./wait-for-postgres.sh"] + # ports: + # - "8000:8000" + # depends_on: + # - db + # links: + # - db:db http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/f30bf2af/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile index d5b8109..776f935 100644 --- a/docker/logstash/Dockerfile +++ b/docker/logstash/Dockerfile @@ -13,14 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM docker.elastic.co/logstash/logstash:5.4.0 +FROM docker.elastic.co/logstash/logstash:5.5.0 MAINTAINER Michelle Beard <msbe...@apache.org> -# Get GeoIP DB -RUN set -x \ - && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \ - && gunzip GeoIP.dat.gz \ - && mkdir -p /usr/share/logstash/GeoIP \ - && mv GeoIP.dat /usr/share/logstash/GeoIP/GeoIP.dat - RUN mkdir /usr/share/logstash/templates \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/f30bf2af/docker/logstash/config/logstash-userale.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/config/logstash-userale.conf b/docker/logstash/config/logstash-userale.conf index 6eb103e..77bee34 100644 --- a/docker/logstash/config/logstash-userale.conf +++ b/docker/logstash/config/logstash-userale.conf @@ -54,8 +54,10 @@ filter { output { # Output data to Elasticsearch instance elasticsearch { - hosts => "elasticsearch:9200" + hosts => "elasticsearch1:9200" index => "userale" + # user => "elastic" + # password => "changeme" document_type => "logs" manage_template => true template_overwrite => true http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/f30bf2af/docker/startup/entrypoint.sh ---------------------------------------------------------------------- diff --git a/docker/startup/entrypoint.sh b/docker/startup/entrypoint.sh index 5686d9d..00953e6 100644 --- a/docker/startup/entrypoint.sh +++ b/docker/startup/entrypoint.sh @@ -17,4 +17,5 @@ # Wait for the Elasticsearch container to be ready before starting Kibana. echo "Stalling for Elasticsearch" -dockerize -wait http://elasticsearch:9200 +dockerize -wait http://elasticsearch1:9200 +dockerize -wait http://elasticsearch2:9200 \ No newline at end of file
