Initial (probably broken) commit of ported docker build
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/1c10c783 Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/1c10c783 Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/1c10c783 Branch: refs/heads/asf-site Commit: 1c10c7835193e5f708a7720bf10e2e85e2a04c19 Parents: da63669 Author: Foley <rgf3...@draper.com> Authored: Wed Mar 22 09:26:17 2017 -0400 Committer: Foley <rgf3...@draper.com> Committed: Wed Mar 22 09:26:17 2017 -0400 ---------------------------------------------------------------------- docker/db/Dockerfile | 21 ++ docker/db/sql.sh | 32 +++ docker/distill/Dockerfile | 50 ++++ docker/distill/distill.conf | 29 +++ docker/distill/distill/__init__.py | 45 ++++ docker/distill/distill/algorithms/__init__.py | 14 ++ .../distill/algorithms/graphs/__init__.py | 14 ++ .../distill/distill/algorithms/graphs/graph.py | 24 ++ .../distill/algorithms/graphs/tests/__init__.py | 22 ++ .../distill/algorithms/stats/__init__.py | 14 ++ docker/distill/distill/algorithms/stats/hist.py | 183 +++++++++++++++ .../distill/algorithms/stats/tests/__init__.py | 22 ++ .../distill/algorithms/tests/__init__.py | 14 ++ docker/distill/distill/app.py | 216 +++++++++++++++++ docker/distill/distill/config.cfg | 57 +++++ docker/distill/distill/models/__init__.py | 14 ++ docker/distill/distill/models/brew.py | 235 +++++++++++++++++++ docker/distill/distill/models/stout.py | 149 ++++++++++++ docker/distill/distill/models/tests/__init__.py | 22 ++ docker/distill/distill/models/userale.py | 137 +++++++++++ docker/distill/distill/server.py | 29 +++ docker/distill/distill/tests/__init__.py | 21 ++ docker/distill/distill/tests/basic_test.py | 24 ++ docker/distill/distill/tests/distill_test.py | 43 ++++ docker/distill/distill/utils/__init__.py | 0 docker/distill/distill/utils/exceptions.py | 25 ++ docker/distill/distill/utils/query_builder.py | 35 +++ docker/distill/distill/utils/tests/__init__.py | 21 ++ docker/distill/distill/utils/validation.py | 39 +++ docker/distill/distill/version.py | 22 ++ docker/distill/requirements.txt | 22 ++ docker/distill/setup.cfg | 30 +++ docker/distill/setup.py | 87 +++++++ docker/docker-compose.yml | 78 ++++++ docker/es/._elasticsearch.yml | Bin 0 -> 222 bytes docker/es/Dockerfile | 5 + docker/es/elasticsearch.yml | 98 ++++++++ docker/kibana/Dockerfile | 7 + docker/kibana/entrypoint.sh | 10 + docker/logstash/Dockerfile | 11 + docker/logstash/config/logstash-apache.conf | 82 +++++++ docker/logstash/config/logstash-userale.conf | 55 +++++ docker/logstash/templates/apache.json | 59 +++++ docker/logstash/templates/userale.json | 157 +++++++++++++ docker/tap/Dockerfile | 63 +++++ docker/tap/README.md | 2 + docker/tap/neon_counts.js | 2 + docker/tap/neon_graph.js | 2 + 48 files changed, 2343 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/db/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile new file mode 100644 index 0000000..bb6152f --- /dev/null +++ b/docker/db/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM postgres:latest +MAINTAINER Michelle Beard <msbe...@apache.org> + +ADD sql.sh docker-entrypoint-initdb.d/sql.sh + +EXPOSE 5432 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/db/sql.sh ---------------------------------------------------------------------- diff --git a/docker/db/sql.sh b/docker/db/sql.sh new file mode 100644 index 0000000..4de31b5 --- /dev/null +++ b/docker/db/sql.sh @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/bin/bash + +echo "Creating POSTGRES DB FROM ENVIRONMENT" +DB_NAME=tapdb +DB_USER=tapuser +DB_PASS="Dr@p3rUs3r" +DB_SERVICE=postgres + +psql -U postgres -c "CREATE USER $DB_USER PASSWORD '$DB_PASS'" +psql -U postgres -c "CREATE DATABASE $DB_NAME OWNER $DB_USER" +psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER" + +#localedef -i de_DE -c -f UTF-8 -A /usr/share/locale/locale.alias de_DE.UTF-8 +##export LANG=en_US.UTF-8 +#locale # confirm that it shows only en_US.UTF-8 for all settings +# finally, run your opennms installer +#/usr/share/opennms/bin/install -l /usr/local/lib -dis http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/distill/Dockerfile b/docker/distill/Dockerfile new file mode 100644 index 0000000..f69edc8 --- /dev/null +++ b/docker/distill/Dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:2 + +# install system wide deps +RUN apt-get -yqq update + +# Set the work directory +RUN mkdir -p /usr/src +WORKDIR /usr/src + +# Install git +##RUN sudo -E apt-get -yqq install \ + ##git + +# Clone Distill +RUN git clone https://github.com/apache/incubator-senssoft-distill.git distill +WORKDIR /usr/src/distill +RUN git pull + + +# Add code +##WORKDIR /app +##ADD setup.py /app + +ADD requirements.txt /distill + +# Install Distill dependencies +RUN pip install -r requirements.txt + +# Expose Ports +EXPOSE 8090 + +# Rest +ADD . /app +RUN python setup.py develop +CMD python distill/server.py \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill.conf ---------------------------------------------------------------------- diff --git a/docker/distill/distill.conf b/docker/distill/distill.conf new file mode 100644 index 0000000..a113dcb --- /dev/null +++ b/docker/distill/distill.conf @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description "Gunicorn application server running Apache Distill" + +# Restart process if it ever fails +start on runlevel [2345] +stop on runlevel [!2345] + +respawn +#Setup user and group that Gunicorn should be run as +setuid nobody +setgid www-data + +# Path to run_server +chdir /path/to/distill +exec gunicorn -c "gunicorn.cfg" scripts/run_server:app http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/__init__.py b/docker/distill/distill/__init__.py new file mode 100644 index 0000000..2b44372 --- /dev/null +++ b/docker/distill/distill/__init__.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask +from elasticsearch_dsl.connections import connections + +# Initialize Flask instance +app = Flask (__name__) + +# Load Configurations +app.config.from_pyfile('config.cfg') + +# Unpack Elasticsearch configuration and create elasticsearch connection +host = app.config ['ES_HOST'] +port = app.config ['ES_PORT'] +http_auth = app.config ['HTTP_AUTH'] +use_ssl = app.config ['USE_SSL'] +verify_certs = app.config ['VERIFY_CERTS'] +ca_certs = app.config ['CA_CERTS'] +client_cert = app.config ['CLIENT_CERT'] +client_key = app.config ['CLIENT_KEY'] +timeout = app.config ['TIMEOUT'] + +# Initialize Elasticsearch instance +es = connections.create_connection (hosts = [host], + port = port, + http_auth = http_auth, + use_ssl = use_ssl, + verify_certs = verify_certs, + ca_certs = ca_certs, + client_cert = client_cert, + client_key = client_key, + timeout=timeout) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/__init__.py b/docker/distill/distill/algorithms/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/docker/distill/distill/algorithms/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/graphs/__init__.py b/docker/distill/distill/algorithms/graphs/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/docker/distill/distill/algorithms/graphs/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/graph.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/graphs/graph.py b/docker/distill/distill/algorithms/graphs/graph.py new file mode 100644 index 0000000..3c44730 --- /dev/null +++ b/docker/distill/distill/algorithms/graphs/graph.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class GraphAnalytics (object): + """ + Distill's graph analytics package. Apply graph algorithms to User Ale log data segmented with + Stout. + """ + + @staticmethod + def foo (): + pass \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/graphs/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/graphs/tests/__init__.py b/docker/distill/distill/algorithms/graphs/tests/__init__.py new file mode 100644 index 0000000..f6f6899 --- /dev/null +++ b/docker/distill/distill/algorithms/graphs/tests/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +distill: tests module. + +Meant for use with py.test. +Organize tests into files, each named xxx_test.py +Read more here: http://pytest.org/ +''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/stats/__init__.py b/docker/distill/distill/algorithms/stats/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/docker/distill/distill/algorithms/stats/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/hist.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/stats/hist.py b/docker/distill/distill/algorithms/stats/hist.py new file mode 100644 index 0000000..b516423 --- /dev/null +++ b/docker/distill/distill/algorithms/stats/hist.py @@ -0,0 +1,183 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distill import es +from distill.utils.query_builder import QueryBuilder +from flask import jsonify +from elasticsearch import Elasticsearch, TransportError + +class Hist (object): + """ + Distill's statistics package. Apply statistical algorithms to User Ale log data segmented with + Stout. Need to query/filter by session or user id. + """ + + def __init__ (self): + # parse out query + pass + + # @staticmethod + # def filter (app, app_type=None, q=''): + + # field = q.get ("field") if q.get ("field") else "" + # size = q.get ("size") if q.get ("size") else 10 + + # query = { "aggs" : { + # "count_by_type" : { + # "filter" : { "term" : { field : }} + # "terms" : { + # "field" : field, + # "size" : 100 + # } + # } + # } + # } + + # d = {} + # # try: + # response = es.search (index=app, doc_type=app_type, body=query) + # # for tag in response['aggregations']['count_by_type']['buckets']: + # # d [tag ['key']] = tag ['doc_count'] + # # except TransportError as e: + # # d ['error'] = e.info + # # except Exception as e: + # # d ['error'] = str (e) + # # return jsonify (d) + # return jsonify (response) + + @staticmethod + def terms (app, app_type=None, q=''): + """ + Group by field (find all elements ) + """ + field = q.get ("field") if q.get ("field") else "" + segment = q.get ("seg") if q.get ("seg") else "*" + size = q.get ("size") if q.get ("size") else 10000 + numhits = q.get ("numhits") if q.get ("numhits") else 10 + + query = { "aggs" : { + "count_by_type" : { + "terms" : { + "field" : field, + "size" : size # maximum number of keys (unique fields) + }, + "aggs" : { + "top" : { # arbitrary name + "top_hits" : { + "size" : numhits, # number of logs in subgroup + "_source" : { # segment on fields - return only subgroup based on field + "include" : [ + segment + ] + } + } + } + } + } + } + } + + d = {} + # try: + response = es.search (index=app, doc_type=app_type, body=query) + # for tag in response['aggregations']['count_by_type']['buckets']: + # d [tag ['key']] = tag ['doc_count'] + # except TransportError as e: + # d ['error'] = e.info + # except Exception as e: + # d ['error'] = str (e) + # return jsonify (d) + return jsonify (response) + + @staticmethod + def unique_terms (app, app_type=None, q=""): + """ + Aggregate the number of unique terms in a field. Missing values are counted and marked as "N/A". + + .. todo:: + + Need to incorporate QueryBuilder library instead of manually generating queries. + + :param app: [string] application name + :param app_type: [string] application type + :param field: [string] field to search against for unique values + :param size: [int] the top size terms returned in the result. Default value is 10. + :param min_hits: [int] return tags which have been found in min_hits or more. Default value is 1. + :return: [dict] dictionary of results + """ + + field = q.get ("field") if q.get ("field") else "" + size = q.get ("size") if q.get ("size") else 10000 + min_hits = q.get ("min_hits") if q.get ("min_hits") else 0 + + print field + query = { "aggs" : { + "terms_agg" : { + "terms" : { + "field" : field, + "size" : size, + "min_doc_count" : min_hits, + "missing" : "N/A" + } + } + } + } + + d = {} + try: + response = es.search (index=app, doc_type=app_type, body=query) + for tag in response['aggregations']['terms_agg']['buckets']: + d [tag ['key']] = tag ['doc_count'] + except TransportError as e: + d ['error'] = e.info + except Exception as e: + d ['error'] = str (e) + return jsonify (d) + + @staticmethod + def histogram (app, app_type=None, q=""): + """ + Only works on numerical data. + """ + field = q.get ("field") if q.get ("field") else "" + + interval = 50 + query = { "aggs" : { + "hist_agg" : { + "histogram" : { + "field" : field, + "interval" : interval + } + } + } + } + + d = {} + try: + response = es.search (index=app, doc_type=app_type, body=query) + for tag in response['aggregations']['hist_agg']['buckets']: + d [tag ['key']] = tag ['doc_count'] + except TransportError as e: + d ['error'] = e.info + except Exception as e: + d ['error'] = str (e) + return jsonify (d) + + def get_value (): + return 0 + + def _parse_msg (query): + # should have form ?measure=name&field=f1, f2&event=a,b + pass http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/stats/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/stats/tests/__init__.py b/docker/distill/distill/algorithms/stats/tests/__init__.py new file mode 100644 index 0000000..f6f6899 --- /dev/null +++ b/docker/distill/distill/algorithms/stats/tests/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +distill: tests module. + +Meant for use with py.test. +Organize tests into files, each named xxx_test.py +Read more here: http://pytest.org/ +''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/algorithms/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/algorithms/tests/__init__.py b/docker/distill/distill/algorithms/tests/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/docker/distill/distill/algorithms/tests/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/app.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/app.py b/docker/distill/distill/app.py new file mode 100644 index 0000000..58434a1 --- /dev/null +++ b/docker/distill/distill/app.py @@ -0,0 +1,216 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from flask import Flask, request, jsonify +from distill import app +from distill.models.brew import Brew +from distill.models.userale import UserAle +from distill.models.stout import Stout +from distill.algorithms.stats.hist import Hist + +@app.route ('/', methods=['GET']) +def index (): + """ + Show Distill version information, connection status, and all registered applications. + + .. code-block:: bash + + $ curl -XGET https://localhost:8090 + + { + "author" : "Michelle Beard", + "email" : "mbe...@draper.com", + "name": "Distill", + "status" : true, + "version" : "1.0", + "applications" : { + "xdata_v3" : { + testing: 205, + parsed: 500, + }, + "test_app" : { + logs: 500, + parsed: 100, + } + } + } + + :return: Distill's status information as JSON blob + """ + return jsonify (name="Distill", version="1.0 alpha", author="Michelle Beard", email="mbe...@draper.com", status=Brew.get_status (), applications=Brew.get_applications ()) + +@app.route ('/create/<app_id>', methods=['POST', 'PUT']) +def create (app_id): + """ + Registers an application in Distill. + + .. code-block:: bash + + $ curl -XPOST https://localhost:8090/xdata_v3 + + :param app_id: Application name + :return: Newly created application's status as JSON blob + """ + return Brew.create (app_id) + +@app.route ('/status/<app_id>', defaults={"app_type" : None}, methods=['GET']) +@app.route ('/status/<app_id>/<app_type>', methods=['GET']) +def status (app_id, app_type): + """ + Presents meta information about an registered application, including field names and document types. + + .. code-block:: bash + + $ curl -XGET https://localhost:8090/status/xdata_v3 + + { + "application": "xdata_v3", + "health": "green", + "num_docs": "433", + "status": "open" + } + + :param app_id: Application name + :return: Registered applications meta data as JSON blob + """ + return Brew.read (app_id, app_type=app_type) + +@app.route ('/update/<app_id>', methods=['POST', 'PUT']) +def update (app_id): + """ + Renames a specific application + + .. code-block:: bash + + $ curl -XPOST https://localhost:8090/update/xdata_v3?name="xdata_v4"; + + :param app_id: Application name + :return: Boolean response message as JSON blob + """ + return Brew.update (app_id) + +@app.route ('/delete/<app_id>', methods=['DELETE']) +def delete (app_id): + """ + Deletes an application permentantly from Distill + + .. code-block:: bash + + $ curl -XDELETE https://localhost:8090/xdata_v3 + + :param app_id: Application name + :return: Boolean response message as JSON blob + """ + return Brew.delete (app_id) + +@app.route ('/search/<app_id>', defaults={"app_type" : None}, methods=['GET']) +@app.route ('/search/<app_id>/<app_type>', methods=['GET']) +def segment (app_id, app_type): + """ + Search against an application on various fields. + + .. code-block:: bash + + $ curl -XGET https://[hostname]:[port]/search/xdata_v3?q=session_id:A1234&size=100&scroll=false&fl=param1,param2 + + :param app_id: Application name + :param app_type: Optional document type to filter against + :param q: Main search query. To return all documents, pass in q=*:* + :param size: Maximum number of documents to return in request + :param scroll: Scroll id if the number of documents exceeds 10,000 + :param fl: List of fields to restrict the result set + :return: JSON blob of result set + """ + q = request.args + return UserAle.segment (app_id, app_type=app_type, params=q) + +@app.route ('/stat/<app_id>', defaults={"app_type" : None}, methods=['GET']) +@app.route ('/stat/<app_id>/<app_type>', methods=['GET']) +def stat (app_id, app_type): + """ + Generic histogram counts for a single registered application filtered optionally by document type. + View the Statistics document page for method definitions and arguments + + .. code-block:: bash + + $ curl -XGET https://localhost:8090/stat/xdata_v3/testing/?stat=terms&elem=signup&event=click + + :param app_id: Application name + :param app_type: Application type + :return: JSON blob of result set + """ + stat = request.args.get ('stat') + q = request.args + + hist_cls = Hist () + method = None + try: + method = getattr (hist_cls, stat) + return method (app_id, app_type, q=q) + except AttributeError: + msg = "Class `{}` does not implement `{}`".format(hist_cls.__class__.__name__, stat) + return jsonify (error=msg) + +@app.route ('/denoise/<app_id>', methods=['GET']) +def denoise (app_id): + """ + Bootstrap script to cleanup the raw logs. A document type called "parsed" + will be stored with new log created unless specified in the request. Have option to save + parsed results back to data store. These parsed logs can be intergrated with STOUT results + by running the stout bootstrap script. + + .. code-block:: bash + + $ curl -XGET https://localhost:8090/denoise/xdata_v3?save=true&type=parsed + + :param app_id: Application name + :return: [dict] + """ + doc_type = 'parsed' + save = False + q = request.args + if 'save' in q: + save = str2bool (q.get ('save')) + if 'type' in q: + # @TODO: Proper cleanup script needs to happen + doc_type = q.get ('type') + return UserAle.denoise (app_id, doc_type=doc_type, save=save) + +@app.route ('/stout', methods=['GET']) +def merge_stout (): + """ + Bootstrap script to aggregate user ale logs to stout master answer table + This will save the merged results back to ES instance at new index stout + OR denoise data first, then merge with the stout index... + If STOUT is enabled, the select method expects a stout index to exist or otherwise + it will return an error message. + + .. code-block:: bash + + $ curl -XGET https://locahost:8090/stout/xdata_v3 + + :return: Status message + """ + flag = app.config ['ENABLE_STOUT'] + if flag: + return Stout.ingest () + return jsonify (status="STOUT is disabled.") + +@app.errorhandler(404) +def page_not_found (error): + """ + Generic Error Message + """ + return "Unable to find Distill." http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/config.cfg ---------------------------------------------------------------------- diff --git a/docker/distill/distill/config.cfg b/docker/distill/distill/config.cfg new file mode 100644 index 0000000..189e3ea --- /dev/null +++ b/docker/distill/distill/config.cfg @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Statement for enabling the development environment +DEBUG = True + +# Host +HOST = '0.0.0.0' + +# Port +PORT = 8090 + +# Enable STOUT integration into Distill +ENABLE_STOUT = False +SQLITEDB = '../path/to/stout/stout.db' +MASTER = '../path/to/master/master_ans.csv' +MAPPINGS = '../path/to/mappings/MOT_Mappings.csv' +SELECTED = '../path/to/stout/selected_vars_for_distill.csv' + +# Elasticsearch Configuration +ES_HOST = 'http://elasticsearch' +ES_PORT = 9200 +HTTP_AUTH = None +USE_SSL = False +VERIFY_CERTS = False +CA_CERTS = None +CLIENT_CERT = None +CLIENT_KEY = None +TIMEOUT = 3 + +# Application threads. A common general assumption is +# using 2 per available processor cores - to handle +# incoming requests using one and performing background +# operations using the other. +THREADS_PER_PAGE = 2 + +# Enable protection agains *Cross-site Request Forgery (CSRF)* +# CSRF_ENABLED = True + +# Use a secure, unique and absolutely secret key for +# signing the data. +# CSRF_SESSION_KEY = "secret" + +# Secret key for signing cookies +# SECRET_KEY = "secret" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/models/__init__.py b/docker/distill/distill/models/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/docker/distill/distill/models/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/brew.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/models/brew.py b/docker/distill/distill/models/brew.py new file mode 100644 index 0000000..28d16b3 --- /dev/null +++ b/docker/distill/distill/models/brew.py @@ -0,0 +1,235 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from elasticsearch import Elasticsearch, TransportError +from flask import jsonify +from distill import es + +class Brew (object): + """ + Distill supports basic CRUD operations and publishes the status + of an persistenct database. Eventually it will support ingesting logs sent from + an registered application. + """ + + @staticmethod + def get_status (): + """ + Fetch the status of the underlying database instance. + + :return: [bool] if connection to database instance has been established + """ + return es.ping (ignore=[400, 404]) + + @staticmethod + def get_applications (): + """ + Fetch all the registered applications in Distill. + + .. note:: Private indexes starting with a period are not included in the result set + + :return: [dict] dictionary of all registered applications and meta information + """ + doc = {} + query = { "aggs" : { + "count_by_type" : { + "terms" : { + "field" : "_type", + "size" : 100 + } + } + } + } + + try: + cluster_status = es.cat.indices (h=["index"], pri=False) + x = cluster_status.splitlines() + + for idx in x: + idx = idx.rstrip () + + # Ignore private indexes (like .kibana or .stout) + if idx [:1] != '.': + response = es.search (index=idx, body=query) + d = {} + for tag in response["aggregations"]["count_by_type"]["buckets"]: + d [tag ['key']] = tag ['doc_count'] + doc [idx] = d + except TransportError as e: + doc ['error'] = e.info + except Exception as e: + doc ['error'] = str (e) + return doc + + @staticmethod + def create (app): + """ + Register a new application in Distill + + .. code-block:: bash + + { + "application" : "xdata_v3", + "health" : "green", + "num_docs" : 0, + "status" : "open" + } + + :param app: [string] application name (e.g. xdata_v3) + :return: [dict] dictionary of application and its meta information + """ + + # ignore 400 cause by IndexAlreadyExistsException when creating an index + res = es.indices.create (index=app, ignore=[400, 404]) + doc = _get_cluster_status (app) + return jsonify (doc) + + @staticmethod + def read (app, app_type=None): + """ + Fetch meta data associated with an application + + .. code-block:: bash + + Example: + { + "application" : "xdata_v3", + "health" : "green", + "num_docs" : "100", + "status" : "open" + "types" : { + "raw_logs" : { + "@timestamp" : "date", + "action" : "string", + "elementId" : "string" + }, + "parsed" : { + "@timestamp" : "date", + "elementId_interval" : "string" + }, + "graph" : { + "uniqueID" : "string", + "transition_count" : "long", + "p_value" : "float" + } + } + } + + :param app: [string] application name (e.g. xdata_v3) + :return: [dict] dictionary of application and its meta information + """ + + return jsonify (_get_cluster_status (app, app_type=app_type)) + + @staticmethod + def update (app): + """ + .. todo:: + Currently not implemented + """ + + return jsonify (status="not implemented") + + @staticmethod + def delete (app): + """ + Technically closes the index so its content is not searchable. + + .. code-block: bash + + Example: + { + status: "Deleted index xdata_v3" + } + + :param app: [string] application name (e.g. xdata_v3) + :return: [dict] status message of the event + """ + + es.indices.close (index=app, ignore=[400, 404]) + return jsonify (status="Deleted index %s" % app) + +def _get_cluster_status (app, app_type=None): + """ + Return cluster status, index health, and document count as string + + @todo figure out how to count individual documents stored at an app_type (currently shows only index count) + :param app: [string] application name (e.g. xdata_v3) + :return: [dict] dictionary of index meta data including field names + """ + + doc = {} + try: + cluster_status = es.cat.indices (index=app, h=["health", "status", "docs.count"], pri=True, ignore=[400, 404]) + v = str (cluster_status).split (" ") + m = ["health", "status", "num_docs"] + doc = dict (zip (m, v)) + # Add back application + doc ["application"] = app + except TransportError as e: + doc ['error'] = e.info + except Exception as e: + doc ['error'] = str (e) + + doc ['fields'] = _get_all_fields (app, app_type) + return doc + +def _parse_mappings (app, app_type=None): + """ + .. todo: + + Need to parse out result set that presents field list and type + """ + + try: + mappings = es.indices.get_mapping (index=app, doc_type=[app_type], ignore=[400, 404]) + # mappings = yaml.safe_load (json.ess (mappings)) + # print json.dumps (mappings [app]["mappings"], indent=4, separators=(',', ': ')) + ignore = ["properties", "format"] + except TransportError as e: + doc ['error'] = e.info + except Exception as e: + doc ['error'] = str (e) + return doc + +def _get_all_fields (app, app_type=None): + """ + Retrieve all possible fields in an application + + :param app: [string] application name (e.g. xdata_v3) + :param app_type: [string] application type (e.g. logs) + :return: [list] list of strings representing the fields names + """ + d = list () + query = { "aggs" : { + "fields" : { + "terms" : { + "field" : "_field_names", + "size" : 100 + } + } + } + } + + try: + response = es.search (index=app, doc_type=app_type, body=query) + for tag in response['aggregations']['fields']['buckets']: + d.append (tag ['key']) + except TransportError as e: + d.append (str (e.info)) + except Exception as e: + d.append (str (e)) + return d http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/stout.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/models/stout.py b/docker/distill/distill/models/stout.py new file mode 100644 index 0000000..d6421d8 --- /dev/null +++ b/docker/distill/distill/models/stout.py @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distill import app, es +from elasticsearch_dsl import DocType, String, Boolean, Date, Nested, Search +from elasticsearch_dsl.query import MultiMatch, Match, Q +from elasticsearch import Elasticsearch, TransportError +from flask import jsonify +import pandas as pd + +class StoutDoc (DocType): + """ + Representation of a Stout documentat. + """ + + sessionID = String (index="not_analyzed") + task1 = Nested () + task2 = Nested () + + class Meta: + index = '.stout' + doc_type = 'testing' + + def save (self, *args, **kwargs): + """ + Save data from parsing as a Stout document in Distill + """ + return super (StoutDoc, self).save (*args, **kwargs) + +class Stout (object): + """ + Main Stout class to support ingest and search operations. + """ + + @staticmethod + def ingest (): + """ + Ingest data coming from Stout to Distill + """ + + # Create the mappings in elasticsearch + StoutDoc.init () + status = True + data = _parse (); + try: + for k,v in data.items (): + doc = StoutDoc () + if 'sessionID' in v: + doc.sessionID = v['sessionID'] + if 'task1' in v: + doc.task1 = v['task1'] + if 'task2' in v: + doc.task2 = v['task2'] + doc.save () + except Error as e: + status = False + return jsonify (status=status) + +def _parse (): + """ + Parse master answer table with mapping into an associative array + + :return: [dict] dictionary of session information + """ + master = app.config ['MASTER'] + mappings = app.config ['MAPPINGS'] + + fileContents=pd.read_csv(master, encoding='utf-8') + plainTextMappings=pd.read_csv(mappings, encoding='raw_unicode_escape') + headers=list(fileContents.columns.values) + + #generate the mapping between header and plain text + translationRow={}; + for fieldIndex in range(1,len(headers)): + t=plainTextMappings.ix[fieldIndex] + translationRow[headers[fieldIndex]]=t[9] + + dictBySessionID={} + translationRow['items.text']='foo' + index=0 + for row in fileContents.iterrows(): + index=index+1 + + taskMetrics={} + index,data=row + identifier=row[1][0].split("::") + sessionID=identifier[0] + taskID=(identifier[1]) + workingData={} + #is this session id already in the dictionary? + if sessionID in dictBySessionID: + #grab the entry as workingData + workingData=dictBySessionID[sessionID] + + sysData={} + task1Data={} + task2Data={} + metaData={} + d={} + + for fieldIndex in range(1,len(headers)): + if not pd.isnull(row[1][fieldIndex]): #only interested in non-null fields + tempDict={} + if headers[fieldIndex] in translationRow: + tempDict['field']=translationRow[headers[fieldIndex]] + #tempDict['field']=translationRow[9] + tempDict['value']=row[1][fieldIndex] + d[headers[fieldIndex]]=row[1][fieldIndex] + if "SYS" in headers[fieldIndex]: + sysData[headers[fieldIndex]]=tempDict + elif "OT1" in headers[fieldIndex]: + task1Data[headers[fieldIndex]]=tempDict + elif "OT2" in headers[fieldIndex]: + task2Data[headers[fieldIndex]]=tempDict + else: + metaData[headers[fieldIndex]]=tempDict + + if d['TSK_TIME_DIFF_']>0: #block tasks with zero time elapsed + a=int(d['TSK_TIME_DIFF_OT1_']) + b=int(d['TSK_TIME_DIFF_OT2_']) + #figure out which task the values belong to + if ((a>0) & (b<=0)): + task1Data['taskID']=taskID + task1Data['meta']=metaData + task1Data['system']=sysData + workingData['task1']=task1Data + elif ((a<=0) & (b>0)): + task2Data['taskID']=taskID + task2Data['meta']=metaData + task2Data['system']=sysData + workingData['task2']=task2Data + else: + raise ValueError('Encountered an unexpected task time diff state') + + workingData['sessionID'] = sessionID + dictBySessionID[sessionID]=workingData + return dictBySessionID http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/models/tests/__init__.py b/docker/distill/distill/models/tests/__init__.py new file mode 100644 index 0000000..f6f6899 --- /dev/null +++ b/docker/distill/distill/models/tests/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +distill: tests module. + +Meant for use with py.test. +Organize tests into files, each named xxx_test.py +Read more here: http://pytest.org/ +''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/models/userale.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/models/userale.py b/docker/distill/distill/models/userale.py new file mode 100644 index 0000000..f63fa51 --- /dev/null +++ b/docker/distill/distill/models/userale.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from elasticsearch import Elasticsearch, TransportError +from elasticsearch_dsl import DocType, String, Boolean, Date, Float, Search +from elasticsearch_dsl.query import MultiMatch, Match, Q +from elasticsearch import Elasticsearch, TransportError +from elasticsearch_dsl.connections import connections +from werkzeug.datastructures import ImmutableMultiDict, MultiDict + +from flask import jsonify, Markup +from distill import app, es +import datetime + +class UserAle (object): + """ + Main method of entry to perform segmentation and integration of STOUT's master + answer table (if STOUT is enabled). Advanced and basic analytics is performed in the + distill.algorithms.stats and distill.algorithms.graphs module. + """ + + @staticmethod + def segment (app, app_type=None, params=''): + """ + Just support match all for now. + """ + q = params.get ("q") if params.get ("q") else {} + fields = params.get ("fields") if params.get ("fields") else [] + size = params.get ("size") if params.get ("size") else 10 + scroll = params.get ("scroll") if params.get ("scroll") else False + fl = params.get ("fl") if params.get ("fl") else [] + + # filters = params.get ("filter") if params.get ("filter") else {} + + # 'q': args.get('q', '{}'), + # 'fields': args.get('fl', '{}'), + # 'size': args.get ('size', 100), + # 'scroll': args.get ('scroll', False), + # 'filters': request_args.getlist ('fq') + query = {} + query ['size'] = size + + if q: + res = q.split(":") + key = res [0] + val = res [1] + query ['query'] = {"match" : { key : val } } + else: + query ['query'] = {"match_all" : {}} + + if len (fields) > 0: + ex = { + "include" : fields.split(",") + } + query ['_source'] = ex + + + response = es.search (index=app, doc_type=app_type, body=query) + + return jsonify (response) + + @staticmethod + def search (app, + app_type=None, + filters=list (), + size=100, + include="*", + scroll=None, + sort_field=None): + """ + Perform a search query. + + :param app: [string] application id (e.g. "xdata_v3") + :param app_type: [string] name of the application type. If None all application types are searched. + :param filters: [list of strings] list of filters for a query. + :param size: [int] maximum number of hits that should be returned + :param sort_field: [string] sorting field. Currently supported fields: "timestamp", "date" + :return: [dict] dictionary with processed results. If STOUT is enabled, STOUT data will be merged with final result. + """ + + # Need some query builder... + log_result = es.search (index=app, doc_type=app_type, body=query, fields=filters, size=size) + + stout_result = Stout.getSessions () + + data = merged_results (log_result, stout_result) + return data + + @staticmethod + def denoise (app, app_type='parsed', save=False): + """ + """ + pass + +""" +Combine a list of dictionaries together to form one complete dictionary +""" +def merge_dicts (lst): + dall = {} + for d in lst: + dall.update (d) + return dall + +""" +Get query parameters from the request and preprocess them. +:param [dict-like structure] Any structure supporting get calls +:result [dict] Parsed parameters +""" +def parse_query_parameters (indx, app_type=None, request_args = {}): + args = {key: value[0] for (key, value) in dict (request_args).iteritems ()} + + # print "args = ", args + # Parse out simple filter queries + filters = [] + for filter in get_all_fields (indx, app_type): + if filter in args: + filters.append((filter, args[filter])) + + return { + 'q': args.get('q', '{}'), + 'fields': args.get('fl', []), + 'size': args.get ('size', 100), + 'scroll': args.get ('scroll', False), + 'filters': request_args.getlist ('fq') + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/server.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/server.py b/docker/distill/distill/server.py new file mode 100644 index 0000000..23acd83 --- /dev/null +++ b/docker/distill/distill/server.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distill import app +from distill.app import * + +""" +Start up a local WSGI server called development +""" +def dev_server (): + host = app.config ['HOST'] + port = app.config ['PORT'] + debug = app.config ['DEBUG'] + app.run (host=host, port=port, debug=debug) + +if __name__ == '__main__': + dev_server () http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/tests/__init__.py b/docker/distill/distill/tests/__init__.py new file mode 100644 index 0000000..09c5e2f --- /dev/null +++ b/docker/distill/distill/tests/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +distill: tests module. + +Meant for use with py.test. +Organize tests into files, each named xxx_test.py +Read more here: http://pytest.org/ +''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/basic_test.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/tests/basic_test.py b/docker/distill/distill/tests/basic_test.py new file mode 100644 index 0000000..712d1fe --- /dev/null +++ b/docker/distill/distill/tests/basic_test.py @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +distill: Test module. + +Meant for use with py.test. +Write each test as a function named test_<something>. +Read more here: http://pytest.org/ +''' + +def test_example(): + assert True http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/tests/distill_test.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/tests/distill_test.py b/docker/distill/distill/tests/distill_test.py new file mode 100644 index 0000000..2fb6502 --- /dev/null +++ b/docker/distill/distill/tests/distill_test.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from flask import Flask, request + +from distill import app as test_app + +def test_example (): + assert True + # with test_app.test_client () as c: + # rv = c.get ('/?tequila=42') + # assert request.args ['tequila'] == '42' + +# import os +# import flaskr +# import unittest +# import tempfile + +# class FlaskrTestCase(unittest.TestCase): + +# def setUp(self): +# self.db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp() +# flaskr.app.config['TESTING'] = True +# self.app = flaskr.app.test_client() +# flaskr.init_db() + +# def tearDown(self): +# os.close(self.db_fd) +# os.unlink(flaskr.app.config['DATABASE']) + +# if __name__ == '__main__': +# unittest.main() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/utils/__init__.py b/docker/distill/distill/utils/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/exceptions.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/utils/exceptions.py b/docker/distill/distill/utils/exceptions.py new file mode 100644 index 0000000..a391241 --- /dev/null +++ b/docker/distill/distill/utils/exceptions.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class Error (Exception): + """Base class for exceptions.""" + pass + +class ValidationError (Error): + """ Exceptions raised for errors in validated a url.""" + + def __init__ (self, url, msg): + self.url = url + self.msg = msg http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/query_builder.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/utils/query_builder.py b/docker/distill/distill/utils/query_builder.py new file mode 100644 index 0000000..017a08b --- /dev/null +++ b/docker/distill/distill/utils/query_builder.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class QueryBuilder (object): + + def __init__ (self, query=None): + if query: + self.query = query + else: + self.query = { + "query" : { + "match_all" : {} + } + } + + + def add_filters (self, filters): + pass + + def add_sorting (self, sort_field='', sort_order=''): + pass + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/tests/__init__.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/utils/tests/__init__.py b/docker/distill/distill/utils/tests/__init__.py new file mode 100644 index 0000000..09c5e2f --- /dev/null +++ b/docker/distill/distill/utils/tests/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' +distill: tests module. + +Meant for use with py.test. +Organize tests into files, each named xxx_test.py +Read more here: http://pytest.org/ +''' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/utils/validation.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/utils/validation.py b/docker/distill/distill/utils/validation.py new file mode 100644 index 0000000..7cd3362 --- /dev/null +++ b/docker/distill/distill/utils/validation.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from distill.utils.exceptions import ValidationError + +def validate_request (q): + """ + Parse out request message and validate inputs + + :param q: Url query string + :raises ValidationError: if the query is missing required parameters + """ + if 'q' not in q: + raise ValidationError ("Missing required parameter: %s" % 'q') + else: + # Handle rest of parsing + pass + +def str2bool (v): + """ + Convert string expression to boolean + + :param v: Input value + :returns: Converted message as boolean type + :rtype: bool + """ + return v.lower() in ("yes", "true", "t", "1") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/distill/version.py ---------------------------------------------------------------------- diff --git a/docker/distill/distill/version.py b/docker/distill/distill/version.py new file mode 100644 index 0000000..6532ea7 --- /dev/null +++ b/docker/distill/distill/version.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Version information for Distill. + +This file is imported by ``Distill.__init__``, +and parsed by ``setup.py``. +""" + +__version__ = "0.1.3" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/requirements.txt ---------------------------------------------------------------------- diff --git a/docker/distill/requirements.txt b/docker/distill/requirements.txt new file mode 100644 index 0000000..5137ad9 --- /dev/null +++ b/docker/distill/requirements.txt @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flask==0.10.1 +#networkx==1.11 +elasticsearch-dsl==2.0.0 +#numpy>=1.10.0 +#scipy>=0.17.0 +pandas>=0.18.1 +pytest>=3.0.0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/setup.cfg ---------------------------------------------------------------------- diff --git a/docker/distill/setup.cfg b/docker/distill/setup.cfg new file mode 100644 index 0000000..08020f4 --- /dev/null +++ b/docker/distill/setup.cfg @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[egg_info] +tag_build = 0.1.4 +tag_svn_revision = false + +[aliases] +test=pytest + +[tool:pytest] +addopts = --verbose --ignore=build --ignore=setup.py --ignore=dist --junitxml=test-report.xml --cov-report xml --cov=distill distill/. +norecursedirs = *.eggs *env* .git + +[build_sphinx] +source-dir = docs +build-dir = docs/_build +all_files = 1 http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/distill/setup.py ---------------------------------------------------------------------- diff --git a/docker/distill/setup.py b/docker/distill/setup.py new file mode 100644 index 0000000..8ddd32f --- /dev/null +++ b/docker/distill/setup.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from setuptools import setup, find_packages +import distutils.cmd +import distutils.log +from setuptools.command.test import test as TestCommand +import io, os, sys, subprocess + +if sys.version_info[:2] < (2, 7): + m = "Python 2.7 or later is required for Distill (%d.%d detected)." + raise ImportError (m % sys.version_info[:2]) + +if sys.argv[-1] == 'setup.py': + print ("To install, run 'python setup.py install'") + print () + +def read (*filenames, **kwargs): + encoding = kwargs.get ('encoding', 'utf-8') + sep = kwargs.get ('sep', '\n') + buf = [] + for filename in filenames: + with io.open (filename, encoding=encoding) as f: + buf.append (f.read ()) + return sep.join (buf) + +# Get the version string +def get_version (): + basedir = os.path.dirname (__file__) + with open (os.path.join (basedir, 'distill/version.py')) as f: + version = {} + exec (f.read (), version) + return version['__version__'] + raise RuntimeError ('No version info found.') + +setup ( + name = "Distill", + version = get_version (), + url = "https://github.com/apache/incubator-senssoft-distill";, + license = "Apache Software License", + author = "Michelle Beard", + author_email = "msbe...@apache.org", + description = "An analytical framework for UserALE.", + long_description = __doc__, + classifiers = [ + 'Development Status :: 4 - Beta', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.7', + 'Natural Language :: English', + 'Environment :: Web Environment', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: OS Independent', + 'Private :: Do Not Upload"' + ], + keywords = "stout userale tap distill", # Separate with spaces + packages = find_packages (exclude=['examples', 'tests']), + include_package_data = True, + zip_safe = False, + setup_requires = ['pytest-runner'], + tests_require = ['pytest>=3.0.0', 'pytest-pylint', 'coverage'], + install_requires = ['Flask==0.10.1', + #'networkx==1.11', + 'elasticsearch-dsl==2.0.0', + #'numpy>=1.10.0', + #'scipy>=0.17.0', + 'pandas>=0.18.1' + ], + entry_points = { + 'console_scripts': [ + 'dev = distill.server:dev_server' + ] + } +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..97ee0cb --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apache TAP Production Build + +version: "2" +services: + # ELK Stack + elasticsearch: + build: ./es + container_name: senssoft-elastic + ports: + - 9200:9200 + - 9300:9300 + volumes: + - ./es/data:/usr/share/elasticsearch/data + - ./es/logs:/usr/share/elasticsearch/logs + - ./es/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml + environment: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" + kibana: + build: ./kibana + container_name: senssoft-kibana + ports: + - 5601:5601 + depends_on: + - "elasticsearch" + userale-forwarder: + build: ./logstash + container_name: senssoft-userale-forwarder + command: -f /etc/logstash/conf.d + ports: + - 8000:8000 + volumes: + - ./logstash/config/logstash-userale.conf:/etc/logstash/conf.d/logstash-userale.conf + - ./logstash/templates/userale.json:/usr/share/logstash/templates/userale.json + depends_on: + - "elasticsearch" + environment: + LS_HEAP_SIZE: "2048m" + distill: + build: + context: . + dockerfile: ./distill/Dockerfile + container_name: distill + ports: + - 8090:8090 + depends_on: + - elasticsearch + links: + - elasticsearch + db: + container_name: tap-db + build: ./db + ports: + - "5432:5432" + tap: + container_name: tap-web + build: ./tap + command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000" + ports: + - "8000:8000" + depends_on: + - db + links: + - db:db \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/._elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/es/._elasticsearch.yml b/docker/es/._elasticsearch.yml new file mode 100644 index 0000000..6da966a Binary files /dev/null and b/docker/es/._elasticsearch.yml differ http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/es/Dockerfile b/docker/es/Dockerfile new file mode 100644 index 0000000..1eb4748 --- /dev/null +++ b/docker/es/Dockerfile @@ -0,0 +1,5 @@ +FROM elasticsearch:5 +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Install XPack +RUN elasticsearch-plugin install --batch x-pack http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/es/elasticsearch.yml ---------------------------------------------------------------------- diff --git a/docker/es/elasticsearch.yml b/docker/es/elasticsearch.yml new file mode 100644 index 0000000..c795b7c --- /dev/null +++ b/docker/es/elasticsearch.yml @@ -0,0 +1,98 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please see the documentation for further information on configuration options: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html> +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: SensSoft +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: soft-01 +# +# Add custom attributes to the node: +# +# node.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: /usr/share/elasticsearch/data +# +# Path to log files: +# +path.logs: /usr/share/elasticsearch/logs +# +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +# bootstrap.mlockall: true +# +# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory +# available on the system and that the owner of the process is allowed to use this limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# Set the bind address to a specific IP (IPv4 or IPv6): +# +network.host: 0.0.0.0 +# +# Set a custom port for HTTP: +# +# http.port: 9200 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html> +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when new node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2"] +# +# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1): +# +discovery.zen.minimum_master_nodes: 1 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html> +# +# ---------------------------------- Gateway ----------------------------------- +# +# Block initial recovery after a full cluster restart until N nodes are started: +# +# gateway.recover_after_nodes: 3 +# +# For more information, see the documentation at: +# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html> +# +# ---------------------------------- Various ----------------------------------- +# +# Disable starting multiple nodes on a single system: +# +# node.max_local_storage_nodes: 1 +# +# Require explicit names when deleting indices: +# +# action.destructive_requires_name: true + +http.cors.enabled : true +http.cors.allow-origin : "*" http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/kibana/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile new file mode 100644 index 0000000..4814f47 --- /dev/null +++ b/docker/kibana/Dockerfile @@ -0,0 +1,7 @@ +FROM kibana:5 +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Install XPack +RUN kibana-plugin install x-pack + +#CMD ["/tmp/entrypoint.sh"] http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/kibana/entrypoint.sh ---------------------------------------------------------------------- diff --git a/docker/kibana/entrypoint.sh b/docker/kibana/entrypoint.sh new file mode 100644 index 0000000..c08d70a --- /dev/null +++ b/docker/kibana/entrypoint.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Wait for the Elasticsearch container to be ready before starting Kibana. +echo "Stalling for Elasticsearch" +while true; do + nc -q 1 elasticsearch 9200 2>/dev/null && break +done + +echo "Starting Kibana" +exec kibana http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile new file mode 100644 index 0000000..3a9533a --- /dev/null +++ b/docker/logstash/Dockerfile @@ -0,0 +1,11 @@ +FROM logstash:5 +MAINTAINER Michelle Beard <msbe...@apache.org> + +# Get GeoIP DB +RUN set -x \ + && wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz \ + && gunzip GeoIP.dat.gz \ + && mkdir -p /usr/share/logstash/GeoIP \ + && mv GeoIP.dat /usr/share/logstash/GeoIP/GeoIP.dat + +RUN mkdir /usr/share/logstash/templates \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/1c10c783/docker/logstash/config/logstash-apache.conf ---------------------------------------------------------------------- diff --git a/docker/logstash/config/logstash-apache.conf b/docker/logstash/config/logstash-apache.conf new file mode 100644 index 0000000..c95882b --- /dev/null +++ b/docker/logstash/config/logstash-apache.conf @@ -0,0 +1,82 @@ +input { + # file { + # path => "/var/log/server_logs/access.log" + # start_position => "beginning" + # sincedb_path => "/dev/null" + # } + + file { + path => [ "/var/log/server_logs/access.log" ] + start_position => "beginning" + sincedb_path => "/dev/null" + type => "apache-access" + } + # file { + # path => "/var/log/server_logs/error.log" + # start_position => "beginning" + # sincedb_path => "/dev/null" + # type => "apache-error" + # } +} + +filter { + # ------------------------ Parse services logs into fields --------------------------- + # APACHE 2 + if [type] == "apache-access" { + # To process log data (message's content) using some regex or precompiled GROK pattern + grok { + match => [ "message", "%{COMBINEDAPACHELOG}"] + } + # To extract log's time according to a date pattern + date { + match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z"] + } + # Extract browser information, if available. + if [agent] != "" { + useragent { + source => "agent" + } + } + # Extract client ip information, if available. + # if [clientip] != "" { + # geoip { + # source => "clientip" + # database => "/usr/share/logstash/GeoIP/GeoIP.dat" + # target => "apache_clientip" + # add_tag => [ "geoip" ] + # } + # } + } + + # if [type] == "apache-error" { + # grok { + # match => [ "message", "%{APACHEERRORLOG}"] + # # Directory where to find the custom patterns + # patterns_dir => ["/etc/logstash/grok"] + # } + # if [clientip] != "" { + # geoip { + # source => "clientip" + # target => "apache_clientip" + # add_tag => [ "geoip" ] + # } + # } + # } +} + +output { + # Output data to Elasticsearch instance + elasticsearch { + hosts => "elasticsearch:9200" + index => "apache" + user => "elastic" + password => "changeme" + manage_template => true + template_overwrite => true + template => "/usr/share/logstash/templates/apache.json" + template_name => "apache" + } + + # Debug + stdout { codec => rubydebug } +}
