This is an automated email from the ASF dual-hosted git repository.
poorejc pushed a commit to branch distill_toolkit_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-flagon-distill.git
The following commit(s) were added to refs/heads/distill_toolkit_refactor by
this push:
new 4a4b2bc [WIP] Distill CRUD examples
4a4b2bc is described below
commit 4a4b2bc9cfada9d0b5c46eea99426fd88f30720a
Author: poorejc <[email protected]>
AuthorDate: Fri Oct 4 10:22:02 2019 -0400
[WIP] Distill CRUD examples
---
CONTRIBUTING.rst | 1 +
MANIFEST.in | 2 +
WIP-distill_examples/elasticDSLtest.ipynb | 392 +++++++++++++++++++++
.../flagon-distill/elasticsearch/elasticsearch.py | 48 +++
requirements.txt | 1 +
setup.cfg | 2 +
setup.py | 2 +
7 files changed, 448 insertions(+)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e6a9f2e..40cc384 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -20,6 +20,7 @@
Contributing to Apache SensSoft
-------------------------------
+##TO DO: Update for WIP-Package
Thank you for contributing to the Distill project!
diff --git a/MANIFEST.in b/MANIFEST.in
index ec629be..03095fb 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,5 @@
+#TO DO: Update for WIP-PACKAGE
+
include MANIFEST.in
include *.rst
include NOTICE LICENSE DISCLAIMER.txt
diff --git a/WIP-distill_examples/elasticDSLtest.ipynb
b/WIP-distill_examples/elasticDSLtest.ipynb
new file mode 100644
index 0000000..f38c4be
--- /dev/null
+++ b/WIP-distill_examples/elasticDSLtest.ipynb
@@ -0,0 +1,392 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<Elasticsearch([{'host': 'localhost'}])>\n"
+ ]
+ }
+ ],
+ "source": [
+ "from elasticsearch import Elasticsearch\n",
+ "from elasticsearch_dsl import connections\n",
+ "from elasticsearch_dsl import Search\n",
+ "from elasticsearch_dsl import Q\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import json\n",
+ "\n",
+ "#create new connection to test instance, given an alias 'flagonTest' for
later reference\n",
+ "#note: connections are easy enough such that examples are enough, no need
for further abstraction\n",
+ "flagonClient = connections.create_connection('flagonTest',
hosts=['localhost'], timeout=60)\n",
+ "\n",
+ "#TODO describe connections\n",
+ "\n",
+ "#hello world test\n",
+ "print(flagonClient)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "21\n"
+ ]
+ }
+ ],
+ "source": [
+ "#create new search object, using cluster alias and index name \n",
+ "#TODO abstract to Query Builder\n",
+ "AleS = Search(using='flagonTest', index=\"userale\") \\\n",
+ " .scan()\n",
+ " \n",
+ "#execute search\n",
+ "response = AleS.execute()\n",
+ "\n",
+ "#print Hello World test for total hits, should match kibana output\n",
+ "print(response.hits.total)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "3\n"
+ ]
+ }
+ ],
+ "source": [
+ "#add query in search parameters, match on \"click\" events\n",
+ "customSearch = AleS.query(\"match\", type=\"click\")\n",
+ "\n",
+ "response = customSearch.execute()\n",
+ "\n",
+ "print(response.hits.total)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'query': {'bool': {'filter': [{'term': {'target': 'a.ui big blue
button'}}], 'must': [{'match': {'type': 'click'}}]}}}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(customSearch.to_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'took': 16, 'timed_out': False, '_shards': {'total': 1, 'successful':
1, 'skipped': 0, 'failed': 0}, 'hits': {'total': 3, 'max_score': 1.8382794,
'hits': [{'_index': 'userale', '_type': 'doc', '_id': 'kJYOV20B7YNHv2dPpmwB',
'_score': 1.8382794, '_source': {'path': ['a.ui big red button', 'div.ui center
aligned text container', 'div.ui very padded vertical segment',
'div.main-wrapper', 'body', 'html', '#document', 'Window'], 'userId': None,
'useraleVersion': 'demo', 'location': {' [...]
+ ]
+ }
+ ],
+ "source": [
+ "print(response.to_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2\n"
+ ]
+ }
+ ],
+ "source": [
+ "#add filter to search params\n",
+ "customSearch2 = AleS.query(\"match\", type=\"click\") \\\n",
+ " .filter(\"term\", target=\"a.ui big blue button\") \\\n",
+ " .scan()\n",
+ "\n",
+ "response = customSearch.execute()\n",
+ "\n",
+ "print(response.hits.total)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'query': {'bool': {'filter': [{'term': {'target': 'a.ui big blue
button'}}], 'must': [{'match': {'type': 'click'}}]}}}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(customSearch.to_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'took': 10, 'timed_out': False, '_shards': {'total': 1, 'successful':
1, 'skipped': 0, 'failed': 0}, 'hits': {'total': 2, 'max_score': 1.8382794,
'hits': [{'_index': 'userale', '_type': 'doc', '_id': 'mJYPV20B7YNHv2dPIGw7',
'_score': 1.8382794, '_source': {'path': ['a.ui big blue button', 'div.ui
center aligned text container', 'div.ui very padded vertical segment',
'div.main-wrapper', 'body', 'html', '#document', 'Window'], 'userId': None,
'useraleVersion': 'demo', 'location': { [...]
+ ]
+ }
+ ],
+ "source": [
+ "print(response.to_dict())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>_id</th>\n",
+ " <th>_index</th>\n",
+ " <th>_score</th>\n",
+ " <th>_source</th>\n",
+ " <th>_type</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>mJYPV20B7YNHv2dPIGw7</td>\n",
+ " <td>userale</td>\n",
+ " <td>1.838279</td>\n",
+ " <td>{'path': ['a.ui big blue button', 'div.ui cent...</td>\n",
+ " <td>doc</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>pJYPV20B7YNHv2dPJWyZ</td>\n",
+ " <td>userale</td>\n",
+ " <td>1.838279</td>\n",
+ " <td>{'path': ['a.ui big blue button', 'div.ui cent...</td>\n",
+ " <td>doc</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " _id _index _score \\\n",
+ "0 mJYPV20B7YNHv2dPIGw7 userale 1.838279 \n",
+ "1 pJYPV20B7YNHv2dPJWyZ userale 1.838279 \n",
+ "\n",
+ " _source _type \n",
+ "0 {'path': ['a.ui big blue button', 'div.ui cent... doc \n",
+ "1 {'path': ['a.ui big blue button', 'div.ui cent... doc "
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#convert responses to dataframe \n",
+ "#TODO extract _source(s) from df \n",
+ "AleData = response.to_dict()\n",
+ "AleMeta = pd.DataFrame.from_dict(AleData[\"hits\"][\"hits\"])\n",
+ "\n",
+ "AleMeta"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'path': ['a.ui big blue button',\n",
+ " 'div.ui center aligned text container',\n",
+ " 'div.ui very padded vertical segment',\n",
+ " 'div.main-wrapper',\n",
+ " 'body',\n",
+ " 'html',\n",
+ " '#document',\n",
+ " 'Window'],\n",
+ " 'userId': '',\n",
+ " 'useraleVersion': 'demo',\n",
+ " 'location': {'x': 657, 'y': 705},\n",
+ " 'toolName': '',\n",
+ " 'toolVersion': '',\n",
+ " 'clientTime': 1569123669151,\n",
+ " 'details': {'alt': False,\n",
+ " 'shift': False,\n",
+ " 'meta': False,\n",
+ " 'clicks': 1,\n",
+ " 'ctrl': False},\n",
+ " '@timestamp': '2019-09-22T03:41:11.890Z',\n",
+ " 'tags': ['_grokparsefailure'],\n",
+ " 'type': 'click',\n",
+ " 'host': '172.18.0.1',\n",
+ " 'target': 'a.ui big blue button',\n",
+ " 'userAction': True}"
+ ]
+ },
+ "execution_count": 57,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#TO DO Make Iterator over source dump into python dataframe\n",
+ "AleLogs =
json.dumps(AleData[\"hits\"][\"hits\"][0]['_source']).replace('null',
'\"\"')\n",
+ "context = defaults.copy()\n",
+ "context.update(user)\n",
+ "json.loads(AleLogs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "path [a.ui big blue button, div.ui center aligned t...\n",
+ "userId \n",
+ "useraleVersion demo\n",
+ "location {'x': 657, 'y': 705}\n",
+ "toolName \n",
+ "toolVersion \n",
+ "clientTime 1569123669151\n",
+ "details {'alt': False, 'shift': False, 'meta': False, ...\n",
+ "@timestamp 2019-09-22T03:41:11.890Z\n",
+ "tags [_grokparsefailure]\n",
+ "type click\n",
+ "host 172.18.0.1\n",
+ "target a.ui big blue button\n",
+ "userAction True\n",
+ "dtype: object\n"
+ ]
+ }
+ ],
+ "source": [
+ "#Pull data into pandas series\n",
+ "AleSeries = pd.read_json(AleLogs, orient='index', typ='series')\n",
+ "print(AleSeries)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "['a.ui big blue button',\n",
+ " 'div.ui center aligned text container',\n",
+ " 'div.ui very padded vertical segment',\n",
+ " 'div.main-wrapper',\n",
+ " 'body',\n",
+ " 'html',\n",
+ " '#document',\n",
+ " 'Window']"
+ ]
+ },
+ "execution_count": 51,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "#Retrieve field\n",
+ "AleSeries['path']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/WIP-distill_package/flagon-distill/elasticsearch/elasticsearch.py
b/WIP-distill_package/flagon-distill/elasticsearch/elasticsearch.py
new file mode 100644
index 0000000..a2886d4
--- /dev/null
+++ b/WIP-distill_package/flagon-distill/elasticsearch/elasticsearch.py
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import Search
+
+flagonClient = connections.create_connection('flagonTest',
hosts=['localhost'], timeout=60)
+
+# create new search object, using cluster alias and index name
+AleS = Search(using='flagonTest', index="userale") \
+#TO DO: Figure out how to integrate query builder
+ .query("match", type="click") \
+ .filter("term", target="a.ui big blue button") \
+ .scan()
+
+# execute search
+response = AleS.execute()
+
+# print Hello World test for total hits, should match kibana output
+print(response.hits.total)
+
+AleS = Search(using=client, index="my-index") \
+ .filter("term", category="search") \
+ .query("match", title="python") \
+ .scan() \
+ .exclude("match", description="beta")
+
+AleS.aggs.bucket('per_tag', 'terms', field='tags') \
+ .metric('max_lines', 'max', field='lines')
+
+response = AleS.execute()
+
+for hit in response:
+ print(hit.meta.score, hit.title)
+
+
diff --git a/requirements.txt b/requirements.txt
index eb7cefe..8b8bfc5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+//TODO Update and Rector for WIP-Package
alabaster==0.7.10
amqp==2.2.1
astroid==1.5.3
diff --git a/setup.cfg b/setup.cfg
index e804958..ceaa64f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+#TO DO: UPDATE FOR WIP-PACKAGE
+
[egg_info]
tag_build = .dev
tag_date = 1
diff --git a/setup.py b/setup.py
index 868e5dd..8bb4418 100644
--- a/setup.py
+++ b/setup.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+##TODO Update for WIP-PACKAGE
+
import setuptools
import io
import os
