Repository: incubator-senssoft
Updated Branches:
  refs/heads/kubernetes 6e27db527 -> bf43699ff


Updated ELK stack to 6.2.2. Seperated cluster instructions.


Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/66639ab5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/66639ab5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/66639ab5

Branch: refs/heads/kubernetes
Commit: 66639ab52e44c81413a2a0c5528ccdd1ee1db97a
Parents: 6e27db5
Author: msbeard <msbe...@apache.org>
Authored: Sun Mar 4 13:45:59 2018 -0500
Committer: msbeard <msbe...@apache.org>
Committed: Sun Mar 4 13:45:59 2018 -0500

----------------------------------------------------------------------
 docker/CLUSTER.md                              |  68 +++++++++
 docker/README.md                               |  45 +++---
 docker/docker-compose.cluster.yaml             | 153 ++++++++++++++++++++
 docker/docker-compose.yml                      | 107 +++++---------
 docker/elasticsearch/Dockerfile                |   6 +-
 docker/kibana/Dockerfile                       |   9 +-
 docker/logstash/Dockerfile                     |  19 +--
 docker/logstash/config/logstash-userale.conf   |  53 -------
 docker/logstash/pipeline/logstash-userale.conf |  52 +++++++
 docker/logstash/templates/userale.json         |  11 +-
 10 files changed, 347 insertions(+), 176 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/CLUSTER.md
----------------------------------------------------------------------
diff --git a/docker/CLUSTER.md b/docker/CLUSTER.md
new file mode 100644
index 0000000..93df592
--- /dev/null
+++ b/docker/CLUSTER.md
@@ -0,0 +1,68 @@
+How to Build SensSoft Docker Containers
+---------------------------------------
+
+1. Install [``Docker``](http://docker.com) on your machine. Require Docker 1.7 
and above.
+
+1. Install ``docker-compose`` in an virtual environment. 
+   Full instructions can be found 
[``here``](https://docs.docker.com/compose/install/).
+   
+   ```bash
+   $ python3 -m venv env
+   $ source env/bin/activate
+   $ pip install -r requirements.txt
+   ```
+
+1. Before launching the Docker containers, ensure your ``vm_max_map_count`` 
+   kernel setting is set to at least 262144.
+   Visit [``Running Elasticsearch in Production 
mode``](https://www.elastic.co/guide/en/elasticsearch/reference/5.5/docker.html#docker-cli-run-prod-mode)
 for OS specific instructions.
+   
+   ```bash
+   # Example for Linux systems
+   $ sysctl -w vm.max_map_count=262144
+   ```
+
+1. Start Elasticsearch cluster:
+    
+    ```bash
+    $ docker-compose -f docker-compose.cluster.yaml up -d --scale 
elasticsearch=3 elasticsearch
+    $ docker-compose -f docker-compose.cluster.yaml up -d loadbalancer
+    ```
+    
+    The loadbalancer node exposes port 9200 on localhost and is the only node 
+    that has HTTP enabled. Services such as Kibana and Logstash connect to the 
+    loadbalancer node directly. Loadbalancer accepts requests from Kibana and 
Logstash 
+    and balances them across the elasticsearch worker nodes. The elasticsearch 
+    worker nodes communicate to each other and the loadbalancer via TCP on 
port 9300. 
+
+    <aside class="warning">
+    Starting an elasticsearch cluster is not recommended on a single server. 
This
+    is just for demonstration purposes only. Please refer to our [Kubernetes] 
guide to 
+    deploy an Elasticsearch cluster.
+    </aside>
+
+1. Confirm cluster state:
+   ```bash
+   $ curl -XGET http://localhost:9200/_cluster/health?pretty
+    {
+     "cluster_name" : "SensSoft",
+     "status" : "green",
+     "timed_out" : false,
+     "number_of_nodes" : 4,
+     "number_of_data_nodes" : 3,
+     "active_primary_shards" : 0,
+     "active_shards" : 0,
+     "relocating_shards" : 0,
+     "initializing_shards" : 0,
+     "unassigned_shards" : 0,
+     "delayed_unassigned_shards" : 0,
+     "number_of_pending_tasks" : 0,
+     "number_of_in_flight_fetch" : 0,
+     "task_max_waiting_in_queue_millis" : 0,
+     "active_shards_percent_as_number" : 100.0
+   }
+   ```
+   Confirm that the `number_of_nodes` is 4 and `number_of_data_nodes` is 3.
+ 
+1. Follow remaining instructions in README.md, starting at #6.
+
+© Copyright 2016 The Charles Stark Draper Laboratory, Inc. All rights 
reserved.

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/README.md
----------------------------------------------------------------------
diff --git a/docker/README.md b/docker/README.md
index 036a995..4281566 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,7 +1,7 @@
 How to Build SensSoft Docker Containers
 ---------------------------------------
 
-1. Install [``Docker``](http://docker.com) on your machine.
+1. Install [``Docker``](http://docker.com) on your machine. Require Docker 1.7 
and above.
 
 1. Install ``docker-compose`` in an virtual environment. 
    Full instructions can be found 
[``here``](https://docs.docker.com/compose/install/).
@@ -21,28 +21,22 @@ How to Build SensSoft Docker Containers
    $ sysctl -w vm.max_map_count=262144
    ```
 
-1. Start Elasticsearch cluster:
-    
-    ```bash
-    $ docker-compose up -d --scale elasticsearch=3 elasticsearch loadbalancer
-    ```
-    
-    The loadbalancer node exposes port 9200 on localhost and is the only node 
-    that has HTTP enabled. Services such as Kibana and Logstash connect to the 
-    loadbalancer node directly. Loadbalancer accepts requests from Kibana and 
Logstash 
-    and balances them across the elasticsearch worker nodes. The elasticsearch 
-    worker nodes communicate to each other and the loadbalancer via TCP on 
port 9300. 
-
-    
-1. Confirm cluster state:
+1. Start Elasticsearch. Give Elasticsearch about 2 minutes to start before 
confirming
+   its state. 
+   
+   ```bash
+   $ docker-compose up -d loadbalancer
+   ```
+
+1. Confirm state:
    ```bash
    $ curl -XGET http://localhost:9200/_cluster/health?pretty
     {
      "cluster_name" : "SensSoft",
      "status" : "green",
      "timed_out" : false,
-     "number_of_nodes" : 4,
-     "number_of_data_nodes" : 3,
+     "number_of_nodes" : 1,
+     "number_of_data_nodes" : 1,
      "active_primary_shards" : 0,
      "active_shards" : 0,
      "relocating_shards" : 0,
@@ -55,7 +49,6 @@ How to Build SensSoft Docker Containers
      "active_shards_percent_as_number" : 100.0
    }
    ```
-   Confirm that the `number_of_nodes` is 4 and `number_of_data_nodes` is 3.
  
 1. Launch logging server. Give Logstash about 2 minutes to start before 
confirming 
    its state.
@@ -105,8 +98,13 @@ How to Build SensSoft Docker Containers
 
    ![alt text][dashboard]
 
+1. To Launch Tap and Distill
+   ```bash
+   $ docker-compose up -d distill tap
+   ```
+   
 1. To stop all containers.
-    ```sh
+    ```bash
     $ docker-compose stop
     ```
  
@@ -123,11 +121,10 @@ Having Issues?
    $ docker-compose logs > err.dump 
    ```
 
-Todo
----- 
-- [ ] TAP docker deployment instructions.
-- [ ] Distill docker deployment instructions. 
-- [ ] Apache SensSoft Docker + Kubernetes instructions.
+TODO
+----
+1. Update Kibana instructions and images. 
+1. Test Kibana visualizations.
 
 [configure_index]: ./images/configure_index.png "Configure Kibana index"
 [confirmation]: ./images/confirmation.png "Confirm index pattern conflicts"

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/docker-compose.cluster.yaml
----------------------------------------------------------------------
diff --git a/docker/docker-compose.cluster.yaml 
b/docker/docker-compose.cluster.yaml
new file mode 100644
index 0000000..5835716
--- /dev/null
+++ b/docker/docker-compose.cluster.yaml
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apache SensSoft Cluster Build
+
+version: "3"
+services:
+
+  # Startup
+  startup:
+    build: ./startup
+    image: senssoft/startup:latest
+    container_name: senssoft-start
+    depends_on:
+      - "loadbalancer"
+    command: bash /opt/entrypoint.sh
+
+  # # Site
+  site:
+    build: ./site
+    image: senssoft/website:latest
+    container_name: senssoft-app
+    ports:
+      - 8080:8080
+    depends_on:
+      - "logstash"
+    command: python -m SimpleHTTPServer 8080
+
+  # Elasticsearch Swarm Workers
+  elasticsearch:
+    build: ./elasticsearch
+    image: senssoft/elasticsearch:6.2.2
+    environment:
+      - NODE_MASTER=true
+      - NUMBER_OF_MASTERS=3
+      - NODE_COORDINATE=true
+      - "DISCOVERY_SERVICE=elasticsearch"
+      - NODE_DATA=true
+      - NODE_INGEST=true
+      - MAX_LOCAL_STORAGE_NODES=1
+      - NETWORK_HOST=0.0.0.0
+      - HTTP_ENABLE=false
+      - HTTP_CORS_ENABLE=false
+      - HTTP_CORS_ALLOW_ORIGIN=*
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    networks:
+      - esnet
+
+  # Elasticsearch Loadbalancer/Entry
+  loadbalancer:
+    build: ./elasticsearch
+    image: senssoft/elasticsearch:6.2.2
+    container_name: senssoft-loadbalancer
+    environment:
+      - NODE_MASTER=false
+      - NUMBER_OF_MASTERS=1
+      - NODE_COORDINATE=true
+      - "DISCOVERY_SERVICE=elasticsearch"
+      - NODE_DATA=false
+      - NODE_INGEST=false
+      - MAX_LOCAL_STORAGE_NODES=1
+      - NETWORK_HOST=0.0.0.0
+      - HTTP_ENABLE=true
+      - HTTP_CORS_ENABLE=true
+      - HTTP_CORS_ALLOW_ORIGIN=*
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ports:
+      - "9200:9200"
+      - "9300"
+    networks:
+      - esnet
+
+  # Kibana
+  kibana:
+    build: ./kibana
+    image: senssoft/kibana:6.2.2
+    container_name: senssoft-kibana
+    environment:
+      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "ELASTICSEARCH_URL=http://loadbalancer:9200";
+    ports:
+      - "5601:5601"
+    depends_on:
+      - "loadbalancer"
+    networks:
+      - esnet
+
+  # Logstash
+  logstash:
+    build: ./logstash
+    image: senssoft/logstash:6.2.2
+    container_name: senssoft-logstash
+    environment:
+      - "ELASTICSEARCH_URL=loadbalancer:9200"
+    ports:
+      - 8100:8100
+    volumes:
+      - 
./logstash/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf
+      - 
./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json
+    depends_on:
+      - "loadbalancer"
+    networks:
+      - esnet
+
+  # Distill
+  distill:
+     build: ./distill
+     image: senssoft/distill:0.1.0
+     container_name: senssoft-distill
+     ports:
+       - 8090:8090
+     depends_on:
+       - "loadbalancer"
+     networks:
+       - esnet
+
+  # TAP's Database
+  db:
+     build: ./db
+     image: senssoft/postgres:latest
+     container_name: senssoft-postgres
+     ports:
+       - "5432:5432"
+
+  # TAP
+  tap:
+     build: ./tap
+     image: senssoft/tap:0.1.0
+     container_name: senssoft-tap
+     # command: ["bash", "./wait-for-postgres.sh"]
+     command: bash -c "/usr/src/app/wait-for-postgres.sh"
+     ports:
+       - "8010:8010"
+     depends_on:
+       - db
+     links:
+       - db:db
+
+networks:
+  esnet:
+    driver: bridge

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index d2b5ca6..25cc8e1 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Apache SensSoft Docker Build
+# Apache SensSoft Single-Node Build
 
 version: "3"
 services:
@@ -21,16 +21,16 @@ services:
   # Startup
   startup:
     build: ./startup
-    image: senssoft.incubator.apache.org/startup:latest
+    image: senssoft/startup:latest
     container_name: senssoft-start
     depends_on:
       - "loadbalancer"
     command: bash /opt/entrypoint.sh
 
-  # Site
+  # # Site
   site:
     build: ./site
-    image: senssoft.incubator.apache.org/website:latest
+    image: senssoft/website:latest
     container_name: senssoft-app
     ports:
       - 8080:8080
@@ -38,43 +38,11 @@ services:
       - "logstash"
     command: python -m SimpleHTTPServer 8080
 
-  # Elasticsearch Workers
-  elasticsearch:
-    build: ./elasticsearch
-    image: senssoft.incubator.apache.org/elasticsearch:5.6.3
-    command:
-      - /bin/sh
-      - -c
-      - |
-          elasticsearch-plugin remove --silent x-pack
-          elasticsearch
-    environment:
-      - NODE_MASTER=true
-      - NUMBER_OF_MASTERS=2
-      - NODE_COORDINATE=true
-      - "DISCOVERY_SERVICE=elasticsearch"
-      - NODE_DATA=true
-      - NODE_INGEST=true
-      - MAX_LOCAL_STORAGE_NODES=1
-      - NETWORK_HOST=0.0.0.0
-      - HTTP_ENABLE=false
-      - HTTP_CORS_ENABLE=false
-      - HTTP_CORS_ALLOW_ORIGIN=*
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    networks:
-      - esnet
-
-  # Elasticsearch Loadbalancer
+  # Elasticsearch Loadbalancer/Entry
   loadbalancer:
     build: ./elasticsearch
-    image: senssoft.incubator.apache.org/elasticsearch:5.6.3
+    image: senssoft/elasticsearch:6.2.2
     container_name: senssoft-loadbalancer
-    command:
-      - /bin/sh
-      - -c
-      - |
-          elasticsearch-plugin remove --silent x-pack
-          elasticsearch
     environment:
       - NODE_MASTER=true
       - NUMBER_OF_MASTERS=1
@@ -97,10 +65,11 @@ services:
   # Kibana
   kibana:
     build: ./kibana
-    image: senssoft.incubator.apache.org/kibana:5.6.3
+    image: senssoft/kibana:6.2.2
     container_name: senssoft-kibana
     environment:
       - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
+      - "ELASTICSEARCH_URL=http://loadbalancer:9200";
     ports:
       - "5601:5601"
     depends_on:
@@ -111,56 +80,54 @@ services:
   # Logstash
   logstash:
     build: ./logstash
-    image: senssoft.incubator.apache.org/logstash:5.6.3
+    image: senssoft/logstash:6.2.2
     container_name: senssoft-logstash
     environment:
       - "ELASTICSEARCH_URL=loadbalancer:9200"
-    command: -f /etc/logstash/conf.d
     ports:
       - 8100:8100
     volumes:
-      - 
./logstash/config/logstash-userale.conf:/etc/logstash/conf.d/logstash-userale.conf
+      - 
./logstash/pipeline/logstash-userale.conf:/usr/share/logstash/pipeline/logstash-userale.conf
       - 
./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json
     depends_on:
       - "loadbalancer"
     networks:
       - esnet
 
-  # Distill
+   # Distill
   distill:
-    build: ./distill
-    image: senssoft.incubator.apache.org/distill:0.1.0
-    container_name: senssoft-distill
-    ports:
-      - 8090:8090
-    depends_on:
-      - "loadbalancer"
-    networks:
-      - esnet
+     build: ./distill
+     image: senssoft/distill:0.1.0
+     container_name: senssoft-distill
+     ports:
+       - 8090:8090
+     depends_on:
+       - "loadbalancer"
+     networks:
+       - esnet
 
-  # TAP's Database
+   # TAP's Database
   db:
-    build: ./db
-    image: senssoft.incubator.apache.org/postgres:latest
-    container_name: senssoft-postgres
-    ports:
-      - "5432:5432"
+     build: ./db
+     image: senssoft/postgres:latest
+     container_name: senssoft-postgres
+     ports:
+       - "5432:5432"
 
   # TAP
   tap:
-    build: ./tap
-    image: senssoft.incubator.apache.org/tap:0.1.0
-    container_name: senssoft-tap
-    # command: ["bash", "./wait-for-postgres.sh"]
-    command: bash -c "/usr/src/app/wait-for-postgres.sh"
-    ports:
-      - "8010:8010"
-    depends_on:
-      - db
-    links:
-      - db:db
+     build: ./tap
+     image: senssoft/tap:0.1.0
+     container_name: senssoft-tap
+     # command: ["bash", "./wait-for-postgres.sh"]
+     command: bash -c "/usr/src/app/wait-for-postgres.sh"
+     ports:
+       - "8010:8010"
+     depends_on:
+       - db
+     links:
+       - db:db
 
 networks:
-
   esnet:
     driver: bridge

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/elasticsearch/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile
index 6256564..f1fe259 100644
--- a/docker/elasticsearch/Dockerfile
+++ b/docker/elasticsearch/Dockerfile
@@ -13,12 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+FROM docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.2
 
 MAINTAINER Michelle Beard <msbe...@apache.org>
 
 # Override config
 COPY config /usr/share/elasticsearch/config
+RUN chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/config
 
 # Set environment
 ENV CLUSTER_NAME SensSoft
@@ -26,7 +27,4 @@ ENV CLUSTER_NAME SensSoft
 # Kubernetes requires swap is turned off
 ENV MEMORY_LOCK false
 
-# DELETE XPACK Plugin
-RUN elasticsearch-plugin remove x-pack --silent
-
 EXPOSE 9200
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/kibana/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
index 91027a5..630fe98 100644
--- a/docker/kibana/Dockerfile
+++ b/docker/kibana/Dockerfile
@@ -13,13 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM docker.elastic.co/kibana/kibana:5.6.3
+FROM docker.elastic.co/kibana/kibana-oss:6.2.2
 
 MAINTAINER Michelle Beard <msbe...@apache.org>
 
-# Override config
-COPY config /usr/share/kibana/config
-
-RUN kibana-plugin remove x-pack
-
-EXPOSE 5601
\ No newline at end of file
+EXPOSE 5601

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile
index fb46e5c..d041e48 100644
--- a/docker/logstash/Dockerfile
+++ b/docker/logstash/Dockerfile
@@ -13,18 +13,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM docker.elastic.co/logstash/logstash:5.6.3
+FROM docker.elastic.co/logstash/logstash-oss:6.2.2
 MAINTAINER Michelle Beard <msbe...@apache.org>
 
-RUN mkdir /usr/share/logstash/templates
-
-RUN logstash-plugin remove x-pack
+# Ensure default logstash configuration is removed
+RUN rm -f /usr/share/logstash/pipeline/logstash.conf
 
-# Known issue: https://github.com/elastic/logstash-docker/issues/45
-RUN sed -i '/xpack/d' /usr/share/logstash/config/logstash.yml
+# Copy Pipeline files over
+ADD pipeline/ /usr/share/logstash/pipeline/
 
-# Copy Configuration files over
-COPY config/logstash-userale.conf /etc/logstash/conf.d/logstash-userale.conf
+# Copy templates over
+RUN mkdir /usr/share/logstash/templates
 COPY templates/userale.json /usr/share/logstash/templates/userale.json
 
-EXPOSE 8100
\ No newline at end of file
+EXPOSE 8100
+
+CMD ["logstash"]
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/logstash/config/logstash-userale.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/config/logstash-userale.conf 
b/docker/logstash/config/logstash-userale.conf
deleted file mode 100644
index 4af6ee7..0000000
--- a/docker/logstash/config/logstash-userale.conf
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-input {
-       http {
-               codec => "json"
-               port => 8100
-               response_headers => { 
-                       "Access-Control-Allow-Origin" => "*" 
-            "Access-Control-Allow-Headers" => "Origin, X-Requested-With, 
Content-Type, Accept"
-        }
-       }
-}
-
-## Add your filters / logstash plugins configuration here
-filter {
-       mutate {
-               remove_field => [ "message", "@version", "headers"]
-       }
-
-       grok {
-       match => [ "useraleVersion", 
"(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ]
-       match => [ "toolVersion", 
"(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ]
-  }
-}
-
-output {
-       # Output data to Elasticsearch instance
-       elasticsearch {
-               hosts => "${ELASTICSEARCH_URL}"
-               index => "userale"
-               document_type => "logs"
-               manage_template => true
-               template_overwrite => true
-               template => "/usr/share/logstash/templates/userale.json"
-               template_name => "userale"
-       }
-
-       # Debug
-       # stdout { codec => rubydebug }
-}

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/logstash/pipeline/logstash-userale.conf
----------------------------------------------------------------------
diff --git a/docker/logstash/pipeline/logstash-userale.conf 
b/docker/logstash/pipeline/logstash-userale.conf
new file mode 100644
index 0000000..8b4313f
--- /dev/null
+++ b/docker/logstash/pipeline/logstash-userale.conf
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+input {
+       http {
+               codec => "json"
+               port => 8100
+               response_headers => { 
+                       "Access-Control-Allow-Origin" => "*" 
+            "Access-Control-Allow-Headers" => "Origin, X-Requested-With, 
Content-Type, Accept"
+        }
+       }
+}
+
+## Add your filters / logstash plugins configuration here
+filter {
+       mutate {
+               remove_field => [ "message", "@version", "headers"]
+       }
+
+       grok {
+       match => [ "useraleVersion", 
"(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ]
+       match => [ "toolVersion", 
"(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ]
+  }
+}
+
+output {
+       # Output data to Elasticsearch instance
+       elasticsearch {
+               hosts => "${ELASTICSEARCH_URL}"
+               index => "userale"
+               manage_template => true
+               template_overwrite => true
+               template => "/usr/share/logstash/templates/userale.json"
+               template_name => "userale"
+       }
+
+       # Debug
+       stdout { codec => rubydebug }
+}

http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/66639ab5/docker/logstash/templates/userale.json
----------------------------------------------------------------------
diff --git a/docker/logstash/templates/userale.json 
b/docker/logstash/templates/userale.json
index c1bdbb1..5aa92d0 100644
--- a/docker/logstash/templates/userale.json
+++ b/docker/logstash/templates/userale.json
@@ -1,5 +1,5 @@
 {
-    "template": "userale",
+    "index_patterns": "userale",
     "order": 1,
     "settings":
     {
@@ -21,14 +21,7 @@
     },
     "mappings":
     {
-        "_default_":
-        {
-            "_all":
-            {
-                "enabled": false
-            }
-        },
-        "logs":
+        "doc":
         {
         "dynamic_templates": [
                 {

Reply via email to