This is an automated email from the ASF dual-hosted git repository.
poorejc pushed a commit to branch distill_toolkit_refactor
in repository https://gitbox.apache.org/repos/asf/incubator-flagon-distill.git
The following commit(s) were added to refs/heads/distill_toolkit_refactor by
this push:
new c788d13 updated examples
c788d13 is described below
commit c788d13db809cde88e26e66cfcb8b984f313f970
Author: poorejc <[email protected]>
AuthorDate: Sun Oct 6 22:00:34 2019 -0400
updated examples
---
WIP-distill_examples/elasticDSLtest.ipynb | 515 +++++++++++++++++-------------
1 file changed, 299 insertions(+), 216 deletions(-)
diff --git a/WIP-distill_examples/elasticDSLtest.ipynb
b/WIP-distill_examples/elasticDSLtest.ipynb
index f38c4be..95a2bd5 100644
--- a/WIP-distill_examples/elasticDSLtest.ipynb
+++ b/WIP-distill_examples/elasticDSLtest.ipynb
@@ -2,8 +2,10 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 10,
- "metadata": {},
+ "execution_count": 20,
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [
{
"name": "stdout",
@@ -21,6 +23,7 @@
"\n",
"import numpy as np\n",
"import pandas as pd\n",
+ "from pandas.io.json import json_normalize\n",
"import json\n",
"\n",
"#create new connection to test instance, given an alias 'flagonTest' for
later reference\n",
@@ -35,147 +38,69 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "21\n"
+ "{'path': ['a.ui big red button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 462, 'y': 688}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123638986, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:40:39.612Z',
'tags': ['_grokparsefailure'], 'type': 'm [...]
+ "{'path': ['a.ui big red button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 462, 'y': 688}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123638986, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:40:39.613Z',
'tags': ['_grokparsefailure'], 'type': 'c [...]
+ "{'path': ['div.ui very padded vertical segment', 'div.main-wrapper',
'body', 'html', '#document', 'Window'], 'userId': None, 'useraleVersion':
'demo', 'location': {'x': 593, 'y': 425}, 'toolName': None, 'toolVersion':
None, 'clientTime': 1569123667069, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'div.ui very padded vertical segment',
'userAction': True}\n",
+ "{'path': ['div.ui center aligned text container', 'div.ui very padded
vertical segment', 'div.main-wrapper', 'body', 'html', '#document', 'Window'],
'userId': None, 'useraleVersion': 'demo', 'location': {'x': 662, 'y': 642},
'toolName': None, 'toolVersion': None, 'clientTime': 1569123667706, 'details':
None, '@timestamp': '2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'],
'type': 'mouseover', 'host': '172.18.0.1', 'target': 'div.ui center aligned
text container', 'userActi [...]
+ "{'path': ['div.ui center aligned text container', 'div.ui very padded
vertical segment', 'div.main-wrapper', 'body', 'html', '#document', 'Window'],
'userId': None, 'useraleVersion': 'demo', 'location': {'x': 722, 'y': 710},
'toolName': None, 'toolVersion': None, 'clientTime': 1569123668022, 'details':
None, '@timestamp': '2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'],
'type': 'mouseover', 'host': '172.18.0.1', 'target': 'div.ui center aligned
text container', 'userActi [...]
+ "{'path': ['div.ui very padded vertical segment', 'div.main-wrapper',
'body', 'html', '#document', 'Window'], 'userId': None, 'useraleVersion':
'demo', 'location': {'x': 747, 'y': 725}, 'toolName': None, 'toolVersion':
None, 'clientTime': 1569123668099, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'div.ui very padded vertical segment',
'userAction': True}\n",
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 657, 'y': 705}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123669095, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:11.890Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': None, 'y': None}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123669108, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'focus',
'host': '172.18.0.1', 'target': 'a.ui big blue button', 'us [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 657, 'y': 705}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123669151, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:11.890Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 615, 'y': 679}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123671030, 'details': None, '@timestamp':
'2019-09-22T03:41:11.891Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'a.ui big blue button', ' [...]
+ "{'path': ['h2.ui header', 'div#main-masthead.ui masthead padded center
aligned inverted vertical segment', 'div.main-wrapper', 'body', 'html',
'#document', 'Window'], 'userId': None, 'useraleVersion': 'demo', 'location':
{'x': 342, 'y': 264}, 'toolName': None, 'toolVersion': None, 'clientTime':
1569123667026, 'details': None, '@timestamp': '2019-09-22T03:41:11.890Z',
'tags': ['_grokparsefailure'], 'type': 'mouseover', 'host': '172.18.0.1',
'target': 'h2.ui header', 'userAction': T [...]
+ "{'path': ['p', 'div.ui center aligned text container', 'div.ui very
padded vertical segment', 'div.main-wrapper', 'body', 'html', '#document',
'Window'], 'userId': None, 'useraleVersion': 'demo', 'location': {'x': 671,
'y': 488}, 'toolName': None, 'toolVersion': None, 'clientTime': 1569123667109,
'details': None, '@timestamp': '2019-09-22T03:41:11.890Z', 'tags':
['_grokparsefailure'], 'type': 'mouseover', 'host': '172.18.0.1', 'target':
'p', 'userAction': True}\n",
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 685, 'y': 681}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123667906, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'a.ui big blue button', ' [...]
+ "{'path': ['a.ui big yellow button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 737, 'y': 717}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123668043, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'a.ui big yellow button [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 672, 'y': 723}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123668459, 'details': None, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'mouseover',
'host': '172.18.0.1', 'target': 'a.ui big blue button', ' [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': None, 'y': None}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123669108, 'details': {'window': True}, '@timestamp':
'2019-09-22T03:41:11.890Z', 'tags': ['_grokparsefailure'], 'type': 'focus',
'host': '172.18.0.1', 'target': 'a.ui big blue [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 657, 'y': 705}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123669151, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:11.890Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
+ "{'path': ['div.ui center aligned text container', 'div.ui very padded
vertical segment', 'div.main-wrapper', 'body', 'html', '#document', 'Window'],
'userId': None, 'useraleVersion': 'demo', 'location': {'x': 704, 'y': 676},
'toolName': None, 'toolVersion': None, 'clientTime': 1569123670207, 'details':
None, '@timestamp': '2019-09-22T03:41:11.891Z', 'tags': ['_grokparsefailure'],
'type': 'mouseover', 'host': '172.18.0.1', 'target': 'div.ui center aligned
text container', 'userActi [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 602, 'y': 694}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123673249, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:13.360Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 602, 'y': 694}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123673161, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:13.360Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
+ "{'path': ['a.ui big blue button', 'div.ui center aligned text
container', 'div.ui very padded vertical segment', 'div.main-wrapper', 'body',
'html', '#document', 'Window'], 'userId': None, 'useraleVersion': 'demo',
'location': {'x': 602, 'y': 694}, 'toolName': None, 'toolVersion': None,
'clientTime': 1569123673249, 'details': {'alt': False, 'shift': False, 'meta':
False, 'clicks': 1, 'ctrl': False}, '@timestamp': '2019-09-22T03:41:13.360Z',
'tags': ['_grokparsefailure'], 'type': ' [...]
]
}
],
"source": [
"#create new search object, using cluster alias and index name \n",
"#TODO abstract to Query Builder\n",
- "AleS = Search(using='flagonTest', index=\"userale\") \\\n",
- " .scan()\n",
- " \n",
- "#execute search\n",
- "response = AleS.execute()\n",
- "\n",
- "#print Hello World test for total hits, should match kibana output\n",
- "print(response.hits.total)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "3\n"
- ]
- }
- ],
- "source": [
- "#add query in search parameters, match on \"click\" events\n",
- "customSearch = AleS.query(\"match\", type=\"click\")\n",
- "\n",
- "response = customSearch.execute()\n",
- "\n",
- "print(response.hits.total)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "{'query': {'bool': {'filter': [{'term': {'target': 'a.ui big blue
button'}}], 'must': [{'match': {'type': 'click'}}]}}}\n"
- ]
- }
- ],
- "source": [
- "print(customSearch.to_dict())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "{'took': 16, 'timed_out': False, '_shards': {'total': 1, 'successful':
1, 'skipped': 0, 'failed': 0}, 'hits': {'total': 3, 'max_score': 1.8382794,
'hits': [{'_index': 'userale', '_type': 'doc', '_id': 'kJYOV20B7YNHv2dPpmwB',
'_score': 1.8382794, '_source': {'path': ['a.ui big red button', 'div.ui center
aligned text container', 'div.ui very padded vertical segment',
'div.main-wrapper', 'body', 'html', '#document', 'Window'], 'userId': None,
'useraleVersion': 'demo', 'location': {' [...]
- ]
- }
- ],
- "source": [
- "print(response.to_dict())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "2\n"
- ]
- }
- ],
- "source": [
- "#add filter to search params\n",
- "customSearch2 = AleS.query(\"match\", type=\"click\") \\\n",
- " .filter(\"term\", target=\"a.ui big blue button\") \\\n",
- " .scan()\n",
+ "AleS = Search(using='flagonTest', index=\"userale\")\n",
"\n",
- "response = customSearch.execute()\n",
- "\n",
- "print(response.hits.total)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "{'query': {'bool': {'filter': [{'term': {'target': 'a.ui big blue
button'}}], 'must': [{'match': {'type': 'click'}}]}}}\n"
- ]
- }
- ],
- "source": [
- "print(customSearch.to_dict())"
+ "#TODO MetaData on Hits\n",
+ "#print Hello World\n",
+ "for hit in AleS.scan():\n",
+ " print(hit.to_dict())\n"
]
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "{'took': 10, 'timed_out': False, '_shards': {'total': 1, 'successful':
1, 'skipped': 0, 'failed': 0}, 'hits': {'total': 2, 'max_score': 1.8382794,
'hits': [{'_index': 'userale', '_type': 'doc', '_id': 'mJYPV20B7YNHv2dPIGw7',
'_score': 1.8382794, '_source': {'path': ['a.ui big blue button', 'div.ui
center aligned text container', 'div.ui very padded vertical segment',
'div.main-wrapper', 'body', 'html', '#document', 'Window'], 'userId': None,
'useraleVersion': 'demo', 'location': { [...]
+ "{'query': {'match': {'type': 'click'}}}\n"
]
}
],
"source": [
- "print(response.to_dict())"
+ "#print query\n",
+ "print(LessAleS.to_dict())"
]
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 65,
"metadata": {},
"outputs": [
{
@@ -199,165 +124,323 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
- " <th>_id</th>\n",
- " <th>_index</th>\n",
- " <th>_score</th>\n",
- " <th>_source</th>\n",
- " <th>_type</th>\n",
+ " <th>@timestamp</th>\n",
+ " <th>clientTime</th>\n",
+ " <th>details_alt</th>\n",
+ " <th>details_clicks</th>\n",
+ " <th>details_ctrl</th>\n",
+ " <th>details_meta</th>\n",
+ " <th>details_shift</th>\n",
+ " <th>host</th>\n",
+ " <th>location_x</th>\n",
+ " <th>location_y</th>\n",
+ " <th>path</th>\n",
+ " <th>tags</th>\n",
+ " <th>target</th>\n",
+ " <th>toolName</th>\n",
+ " <th>toolVersion</th>\n",
+ " <th>type</th>\n",
+ " <th>userAction</th>\n",
+ " <th>userId</th>\n",
+ " <th>useraleVersion</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
- " <td>mJYPV20B7YNHv2dPIGw7</td>\n",
- " <td>userale</td>\n",
- " <td>1.838279</td>\n",
- " <td>{'path': ['a.ui big blue button', 'div.ui cent...</td>\n",
- " <td>doc</td>\n",
+ " <td>2019-09-22T03:41:11.890Z</td>\n",
+ " <td>1569123669151</td>\n",
+ " <td>False</td>\n",
+ " <td>1</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " <td>172.18.0.1</td>\n",
+ " <td>657</td>\n",
+ " <td>705</td>\n",
+ " <td>[a.ui big blue button, div.ui center aligned t...</td>\n",
+ " <td>[_grokparsefailure]</td>\n",
+ " <td>a.ui big blue button</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>click</td>\n",
+ " <td>True</td>\n",
+ " <td>None</td>\n",
+ " <td>demo</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
- " <td>pJYPV20B7YNHv2dPJWyZ</td>\n",
- " <td>userale</td>\n",
- " <td>1.838279</td>\n",
- " <td>{'path': ['a.ui big blue button', 'div.ui cent...</td>\n",
- " <td>doc</td>\n",
+ " <td>2019-09-22T03:41:13.360Z</td>\n",
+ " <td>1569123673249</td>\n",
+ " <td>False</td>\n",
+ " <td>1</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " <td>172.18.0.1</td>\n",
+ " <td>602</td>\n",
+ " <td>694</td>\n",
+ " <td>[a.ui big blue button, div.ui center aligned t...</td>\n",
+ " <td>[_grokparsefailure]</td>\n",
+ " <td>a.ui big blue button</td>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " <td>click</td>\n",
+ " <td>True</td>\n",
+ " <td>None</td>\n",
+ " <td>demo</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
- " _id _index _score \\\n",
- "0 mJYPV20B7YNHv2dPIGw7 userale 1.838279 \n",
- "1 pJYPV20B7YNHv2dPJWyZ userale 1.838279 \n",
+ " @timestamp clientTime details_alt
details_clicks \\\n",
+ "0 2019-09-22T03:41:11.890Z 1569123669151 False
1 \n",
+ "1 2019-09-22T03:41:13.360Z 1569123673249 False
1 \n",
"\n",
- " _source _type \n",
- "0 {'path': ['a.ui big blue button', 'div.ui cent... doc \n",
- "1 {'path': ['a.ui big blue button', 'div.ui cent... doc "
+ " details_ctrl details_meta details_shift host location_x
\\\n",
+ "0 False False False 172.18.0.1 657
\n",
+ "1 False False False 172.18.0.1 602
\n",
+ "\n",
+ " location_y path
\\\n",
+ "0 705 [a.ui big blue button, div.ui center aligned t... \n",
+ "1 694 [a.ui big blue button, div.ui center aligned t... \n",
+ "\n",
+ " tags target toolName toolVersion
type \\\n",
+ "0 [_grokparsefailure] a.ui big blue button None None
click \n",
+ "1 [_grokparsefailure] a.ui big blue button None None
click \n",
+ "\n",
+ " userAction userId useraleVersion \n",
+ "0 True None demo \n",
+ "1 True None demo "
]
},
- "execution_count": 17,
+ "execution_count": 65,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "#convert responses to dataframe \n",
- "#TODO extract _source(s) from df \n",
- "AleData = response.to_dict()\n",
- "AleMeta = pd.DataFrame.from_dict(AleData[\"hits\"][\"hits\"])\n",
+ "#add query, filter to search params\n",
+ "FilterAleS = AleS \\\n",
+ " .query(\"match\", type=\"click\") \\\n",
+ " .filter(\"term\", target=\"a.ui big blue button\")\n",
+ "\n",
+ "#print into pd df\n",
+ "AleList = []\n",
+ "for hit in FilterAleS.scan():\n",
+ " AleList.append(hit.to_dict())\n",
+ " \n",
+ "AleDf = json_normalize(AleList, sep=\"_\")\n",
"\n",
- "AleMeta"
+ "AleDf"
]
},
{
"cell_type": "code",
- "execution_count": 57,
+ "execution_count": 66,
"metadata": {},
"outputs": [
{
"data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>0</th>\n",
+ " <th>1</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>@timestamp</th>\n",
+ " <td>2019-09-22T03:41:11.890Z</td>\n",
+ " <td>2019-09-22T03:41:13.360Z</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>clientTime</th>\n",
+ " <td>1569123669151</td>\n",
+ " <td>1569123673249</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>details_alt</th>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>details_clicks</th>\n",
+ " <td>1</td>\n",
+ " <td>1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>details_ctrl</th>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>details_meta</th>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>details_shift</th>\n",
+ " <td>False</td>\n",
+ " <td>False</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>host</th>\n",
+ " <td>172.18.0.1</td>\n",
+ " <td>172.18.0.1</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>location_x</th>\n",
+ " <td>657</td>\n",
+ " <td>602</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>location_y</th>\n",
+ " <td>705</td>\n",
+ " <td>694</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>path</th>\n",
+ " <td>[a.ui big blue button, div.ui center aligned t...</td>\n",
+ " <td>[a.ui big blue button, div.ui center aligned t...</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>tags</th>\n",
+ " <td>[_grokparsefailure]</td>\n",
+ " <td>[_grokparsefailure]</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>target</th>\n",
+ " <td>a.ui big blue button</td>\n",
+ " <td>a.ui big blue button</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>toolName</th>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>toolVersion</th>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>type</th>\n",
+ " <td>click</td>\n",
+ " <td>click</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>userAction</th>\n",
+ " <td>True</td>\n",
+ " <td>True</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>userId</th>\n",
+ " <td>None</td>\n",
+ " <td>None</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>useraleVersion</th>\n",
+ " <td>demo</td>\n",
+ " <td>demo</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
"text/plain": [
- "{'path': ['a.ui big blue button',\n",
- " 'div.ui center aligned text container',\n",
- " 'div.ui very padded vertical segment',\n",
- " 'div.main-wrapper',\n",
- " 'body',\n",
- " 'html',\n",
- " '#document',\n",
- " 'Window'],\n",
- " 'userId': '',\n",
- " 'useraleVersion': 'demo',\n",
- " 'location': {'x': 657, 'y': 705},\n",
- " 'toolName': '',\n",
- " 'toolVersion': '',\n",
- " 'clientTime': 1569123669151,\n",
- " 'details': {'alt': False,\n",
- " 'shift': False,\n",
- " 'meta': False,\n",
- " 'clicks': 1,\n",
- " 'ctrl': False},\n",
- " '@timestamp': '2019-09-22T03:41:11.890Z',\n",
- " 'tags': ['_grokparsefailure'],\n",
- " 'type': 'click',\n",
- " 'host': '172.18.0.1',\n",
- " 'target': 'a.ui big blue button',\n",
- " 'userAction': True}"
+ " 0
\\\n",
+ "@timestamp 2019-09-22T03:41:11.890Z
\n",
+ "clientTime 1569123669151
\n",
+ "details_alt False
\n",
+ "details_clicks 1
\n",
+ "details_ctrl False
\n",
+ "details_meta False
\n",
+ "details_shift False
\n",
+ "host 172.18.0.1
\n",
+ "location_x 657
\n",
+ "location_y 705
\n",
+ "path [a.ui big blue button, div.ui center aligned t...
\n",
+ "tags [_grokparsefailure]
\n",
+ "target a.ui big blue button
\n",
+ "toolName None
\n",
+ "toolVersion None
\n",
+ "type click
\n",
+ "userAction True
\n",
+ "userId None
\n",
+ "useraleVersion demo
\n",
+ "\n",
+ " 1 \n",
+ "@timestamp 2019-09-22T03:41:13.360Z \n",
+ "clientTime 1569123673249 \n",
+ "details_alt False \n",
+ "details_clicks 1 \n",
+ "details_ctrl False \n",
+ "details_meta False \n",
+ "details_shift False \n",
+ "host 172.18.0.1 \n",
+ "location_x 602 \n",
+ "location_y 694 \n",
+ "path [a.ui big blue button, div.ui center aligned t... \n",
+ "tags [_grokparsefailure] \n",
+ "target a.ui big blue button \n",
+ "toolName None \n",
+ "toolVersion None \n",
+ "type click \n",
+ "userAction True \n",
+ "userId None \n",
+ "useraleVersion demo "
]
},
- "execution_count": 57,
+ "execution_count": 66,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "#TO DO Make Iterator over source dump into python dataframe\n",
- "AleLogs =
json.dumps(AleData[\"hits\"][\"hits\"][0]['_source']).replace('null',
'\"\"')\n",
- "context = defaults.copy()\n",
- "context.update(user)\n",
- "json.loads(AleLogs)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 50,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "path [a.ui big blue button, div.ui center aligned t...\n",
- "userId \n",
- "useraleVersion demo\n",
- "location {'x': 657, 'y': 705}\n",
- "toolName \n",
- "toolVersion \n",
- "clientTime 1569123669151\n",
- "details {'alt': False, 'shift': False, 'meta': False, ...\n",
- "@timestamp 2019-09-22T03:41:11.890Z\n",
- "tags [_grokparsefailure]\n",
- "type click\n",
- "host 172.18.0.1\n",
- "target a.ui big blue button\n",
- "userAction True\n",
- "dtype: object\n"
- ]
- }
- ],
- "source": [
- "#Pull data into pandas series\n",
- "AleSeries = pd.read_json(AleLogs, orient='index', typ='series')\n",
- "print(AleSeries)"
+ "#Or in Series-like object\n",
+ "AleDf.transpose()"
]
},
{
"cell_type": "code",
- "execution_count": 51,
+ "execution_count": 69,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "['a.ui big blue button',\n",
- " 'div.ui center aligned text container',\n",
- " 'div.ui very padded vertical segment',\n",
- " 'div.main-wrapper',\n",
- " 'body',\n",
- " 'html',\n",
- " '#document',\n",
- " 'Window']"
+ "0 a.ui big blue button\n",
+ "1 a.ui big blue button\n",
+ "Name: target, dtype: object"
]
},
- "execution_count": 51,
+ "execution_count": 69,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "#Retrieve field\n",
- "AleSeries['path']"
+ "AleDf.target"
]
},
{
