This is an automated email from the ASF dual-hosted git repository. poorejc pushed a commit to branch distill_toolkit_refactor in repository https://gitbox.apache.org/repos/asf/incubator-flagon-distill.git
commit 6d07d65ad96d8763a9933254240fcdf5216feebc Author: poorejc <[email protected]> AuthorDate: Sat Sep 14 01:42:42 2019 -0400 [FLAGON-459] Created new branch for Distill refactor --- README.rst | 2 +- WIP-distill_package/.gitignore | 64 +++ WIP-distill_package/DISCLAIMER | 7 + WIP-distill_package/LICENSE | 201 ++++++++ WIP-distill_package/MANIFEST.in | 11 + WIP-distill_package/NOTICE | 6 + WIP-distill_package/README.md | 6 + WIP-distill_package/doap-distill.rdf | 57 +++ WIP-distill_package/flagon-distill/__init__.py | 41 ++ .../flagon-distill/graphs/__init__.py | 14 + WIP-distill_package/flagon-distill/graphs/graph.py | 549 +++++++++++++++++++++ WIP-distill_package/flagon-distill/version.py | 22 + WIP-distill_package/setup.cfg | 37 ++ WIP-distill_package/setup.py | 157 ++++++ 14 files changed, 1173 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ca0929e..0715bd5 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ Apache Flagon Distill ======================= -NOTE: This project is not currently maintained and will be refactored. If you are interested in Distill, join the discussion on our dev list: [email protected]. +NOTE: This project is not currently maintained and is being refactored. If you are interested in Distill, join the discussion on our dev list: [email protected]. Distill is part of the `Apache Flagon <http://flagon.incubator.apache.org/>`_ project. It comprises an analytical engine for `UserALE <http://flagon.incubator.apache.org/userale/>`_ diff --git a/WIP-distill_package/.gitignore b/WIP-distill_package/.gitignore new file mode 100644 index 0000000..409b2e0 --- /dev/null +++ b/WIP-distill_package/.gitignore @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.DS_Store +.idea + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# setup_tools +setup_tools + +# Flask instance folder +instance/ + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Distribution / packaging +.Python +env*/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +/bin/ diff --git a/WIP-distill_package/DISCLAIMER b/WIP-distill_package/DISCLAIMER new file mode 100755 index 0000000..580f8e8 --- /dev/null +++ b/WIP-distill_package/DISCLAIMER @@ -0,0 +1,7 @@ +Apache Flagon is an effort undergoing incubation at The Apache Software Foundation (ASF), +sponsored by the Apache Incubator project. Incubation is required of all newly accepted +projects until a further review indicates that the infrastructure, communications, and +decision making process have stabilized in a manner consistent with other successful +ASF projects. While incubation status is not necessarily a reflection of the +completeness or stability of the code, it does indicate that the project has yet to be +fully endorsed by the ASF. diff --git a/WIP-distill_package/LICENSE b/WIP-distill_package/LICENSE new file mode 100755 index 0000000..b38f054 --- /dev/null +++ b/WIP-distill_package/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/WIP-distill_package/MANIFEST.in b/WIP-distill_package/MANIFEST.in new file mode 100644 index 0000000..ec629be --- /dev/null +++ b/WIP-distill_package/MANIFEST.in @@ -0,0 +1,11 @@ +include MANIFEST.in +include *.rst +include NOTICE LICENSE DISCLAIMER.txt +include setup.py setup.cfg + +# Include all sources +graft distill + +# Exclude +global-exclude *.py[co] +global-exclude __pycache__ \ No newline at end of file diff --git a/WIP-distill_package/NOTICE b/WIP-distill_package/NOTICE new file mode 100755 index 0000000..ed6a1a7 --- /dev/null +++ b/WIP-distill_package/NOTICE @@ -0,0 +1,6 @@ +Apache Flagon Distill +Copyright 2019 The Apache Software Foundation + +This product includes software developed at The Apache Software Foundation (http://www.apache.org/), +based on source code originally developed and generously granted to The Apache Software Foundation +by The Charles Stark Draper Laboratory, Inc (http://www.draper.com/). diff --git a/WIP-distill_package/README.md b/WIP-distill_package/README.md new file mode 100644 index 0000000..5d73d29 --- /dev/null +++ b/WIP-distill_package/README.md @@ -0,0 +1,6 @@ +Apache Flagon Distill +-------------------------------------- + +### This is a WIP and is not currently functional. This is part of our refactoring of Apache Distill. Stay tuned... + +Distill is part of the [Apache Flagon](http://flagon.incubator.apache.org/) project. It comprises an analytical engine for [UserALE.js](http://flagon.incubator.apache.org/useralejs) to interact with user activity logs and apply basic analytical operations to the data. \ No newline at end of file diff --git a/WIP-distill_package/doap-distill.rdf b/WIP-distill_package/doap-distill.rdf new file mode 100755 index 0000000..d17fbc1 --- /dev/null +++ b/WIP-distill_package/doap-distill.rdf @@ -0,0 +1,57 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl"?> +<rdf:RDF xml:lang="en" + xmlns="http://usefulinc.com/ns/doap#"; + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + xmlns:asfext="http://projects.apache.org/ns/asfext#"; + xmlns:foaf="http://xmlns.com/foaf/0.1/";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <Project rdf:about="http://flagon.incubator.apache.org/distill/";> + <created>2016-11-10</created> + <license rdf:resource="http://usefulinc.com/doap/licenses/asl20"; /> + <name>Apache Flagon - Distill.js</name> + <homepage rdf:resource="http://flagon.incubator.apache.org"; /> + <asfext:pmc rdf:resource="http://flagon.incubator.apache.org"; /> + <shortdesc>Distill is an analytical framework for UserALE.js</shortdesc> + <description>Distill comprises an analytical engine for UserALE.js to interact with user activity logs and apply basic analytical operations to the data</description> + <bug-database rdf:resource="https://issues.apache.org/jira/browse/FLAGON"; /> + <mailing-list rdf:resource="https://cwiki.apache.org/confluence/x/whTiAw"; /> + <download-page rdf:resource="https://cwiki.apache.org/confluence/x/whTiAw"; /> + <programming-language>Python</programming-language> + <category rdf:resource="https://projects.apache.org/projects.html?category#Library"; /> + <release> + <Version> + <name>0.1.0 release</name> + <created>2019-09-13</created> + <revision>0.0.0</revision> + </Version> + </release> + <repository> + <SVNRepository> + <location rdf:resource="https://git-wip-us.apache.org/repos/asf/incubator-flagon-distill.git"/> + <browse rdf:resource="https://github.com/apache/incubator-flagon-distill"/> + </SVNRepository> + </repository> + <maintainer> + <foaf:Person> + <foaf:name>Flagon Development Team</foaf:name> + <foaf:mbox rdf:resource="mailto:[email protected]"/> + </foaf:Person> + </maintainer> + </Project> +</rdf:RDF> \ No newline at end of file diff --git a/WIP-distill_package/flagon-distill/__init__.py b/WIP-distill_package/flagon-distill/__init__.py new file mode 100644 index 0000000..aa24b84 --- /dev/null +++ b/WIP-distill_package/flagon-distill/__init__.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from elasticsearch_dsl.connections import connections + +from distill.config.elasticsearch import ELASTICSEARCH_PARAMS + + +# Unpack Elasticsearch configuration and create elasticsearch connection +host = ELASTICSEARCH_PARAMS['host'] +port = ELASTICSEARCH_PARAMS['port'] +http_auth = ELASTICSEARCH_PARAMS['http_auth'] +use_ssl = ELASTICSEARCH_PARAMS['use_ssl'] +verify_certs = ELASTICSEARCH_PARAMS['verify_certs'] +ca_certs = ELASTICSEARCH_PARAMS['ca_certs'] +client_cert = ELASTICSEARCH_PARAMS['client_cert'] +client_key = ELASTICSEARCH_PARAMS['client_key'] +timeout = ELASTICSEARCH_PARAMS['timeout'] + +# Initialize Elasticsearch instance +es = connections.create_connection(hosts=host, + port=port, + http_auth=http_auth, + use_ssl=use_ssl, + verify_certs=verify_certs, + ca_certs=ca_certs, + client_cert=client_cert, + client_key=client_key, + timeout=timeout) diff --git a/WIP-distill_package/flagon-distill/graphs/__init__.py b/WIP-distill_package/flagon-distill/graphs/__init__.py new file mode 100644 index 0000000..6acb5d1 --- /dev/null +++ b/WIP-distill_package/flagon-distill/graphs/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/WIP-distill_package/flagon-distill/graphs/graph.py b/WIP-distill_package/flagon-distill/graphs/graph.py new file mode 100755 index 0000000..174e636 --- /dev/null +++ b/WIP-distill_package/flagon-distill/graphs/graph.py @@ -0,0 +1,549 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import json +import collections + +from elasticsearch import helpers + +from distill import es + +class GraphAnalytics (object): + """ + Distill's graph analytics package. Apply graph algorithms + to User Ale log data. + """ + + @staticmethod + def generate_graph(app, + app_type='logs', + log_type='raw', + targets=[], + events=[], + time_range=['now-1h', 'now'], + size=20): + """ + Return all elements from an application, possible matching against + a specific event type (e.g. click, mouseover, etc) + """ + # @TODO ref_url filter + + must_not_query = [ + { + "term": { + "type": "mousedown" + } + }, + { + "term": { + "type": "mouseup" + } + } + ] + + filter_query = [ + { + "term": { + "logType": log_type + }, + }, + ] + + # Filtering + should_query = [] + must_query = [] + + # Include these events in the request + if events: + include_events = { + "terms": { + "type": events + } + } + filter_query.append(include_events) + + target_in = targets[0] + target_out = targets[1] + + if target_in: + include_targets = { + "terms": { + "target": target_in + } + } + + filter_query.append(include_targets) + + # Remove these elementids from result set + for target in target_out: + res = { + "term": { + "target": target + } + } + must_not_query.append(res) + + # Finish off should query + # must_query.append({"bool": {"should": should_query}}) + + # Sort By Time + sort_query = [ + { + "clientTime": { + "order": "asc" + } + } + ] + + # Timestamp range - date math + timestamp_query = { + "range": { + "@timestamp": { + "gte": time_range[0], + "lte": time_range[1] + } + } + } + filter_query.append(timestamp_query) + + agg_query = dict() + + # Get all unique sessions + session_query = { + "terms": { + "field": "sessionID", + "min_doc_count": 1 + } + } + + agg_query['sessions'] = session_query + + # Generating all top targets and breakdowns by type, including path_length + target_query = { + "terms": { + "field": "target", + "min_doc_count": 1, + "size": size + }, + "aggs": { + "events": { + "terms": { + "field": "type", + "min_doc_count": 1, + "size": size + } + }, + "top_target": { + "top_hits": { + "script_fields": { + "path_length": { + "script": { + "lang": "painless", + "inline": "doc['path'].length;" + } + } + }, + "size": 1 + } + } + } + } + + agg_query['targets'] = target_query + + # Main query + query = { + "sort": sort_query, + "query": { + "bool": { + # "must": must_query, + # "should": should_query, + "filter": filter_query, + "must_not": must_not_query, + # "minimum_should_match": len(should_query) - 1 + } + }, + "_source": { + "includes": ['*'], + }, + "script_fields": { + "path_length": { + "script": { + "lang": "painless", + "inline": "doc['path'].length;" + } + } + }, + "aggregations": agg_query + } + + # return query + # Process Aggregate Results + response = es.search(app, doc_type=app_type, body=query, size=0) + # Only want to look at aggregations + sessions = response['aggregations']['sessions']['buckets'] + # allSessions = { x['key']: [] for x in sessions } + # intervalSessions = { x['key']: [] for x in sessions } + + # Deal with bar chart + allTargets = response['aggregations']['targets']['buckets'] + + # Re-execute query to get all hits + iter = helpers.scan(es, + query=query, + index=app, + doc_type=app_type, + preserve_order=True) + + allSessions = dict() + # Store all hits in the user's bucket. + for elem in iter: + data = elem['_source'] + data['pathLength'] = elem['fields']['path_length'][0] + if 'sessionID' in data: + sessionID = data['sessionID'] + if sessionID in allSessions: + allSessions[sessionID].append(data) + else: + allSessions[sessionID] = [data] + + # This fixed sequence/interval logging that what was produced in + # UserALE.js v 0.2.0 + # Possible to remove self-loops here as well (html->html->html->window) := (html->window) + intervalSessions = dict() + for sessionID in allSessions: + data = allSessions[sessionID] + newData = [] + intervalLog = [] + pairs = zip(data, data[1:]) + + for curr, next in pairs: + target1 = curr['target'] + event1 = curr['type'] + target2 = next['target'] + event2 = next['type'] + if target1 != target2: # ignore self-loops + targetChange = int(True) + eventChange = int(False) + if event1 != event2: + eventChange = int(True) + + # Starting over no matter what + # Based off of curr, update the log + curr['targetChange'] = targetChange + curr['typeChange'] = eventChange + curr['intervalCount'] = len(intervalLog) # some number maybe 0 + if len(intervalLog) >= 2: + # Calculate duration + curr['duration'] = intervalLog[-1:]['clientTime'] - \ + intervalLog[0]['clientTime'] + else: + curr['duration'] = 0 + newData.append(curr) + intervalLog = [] + # else: + # # They are the same + # targetChange = int(False) + # eventChange = int(False) + # if event1 != event2: + # eventChange = int(True) + # # starting over + # curr['targetChange'] = targetChange + # curr['typeChange'] = eventChange + # curr['intervalCount'] = len(intervalLog) + # # if len(intervalLog) >= 2: + # # # Calculate duration + # # curr['duration'] = intervalLog[-1:]['clientTime'] - \ + # # intervalLog[0]['clientTime'] + # # else: + # # curr['duration'] = 0 + # newData.append(curr) + # intervalLog = [] + # else: + # # increase counter + # intervalLog.append(curr) + intervalSessions[sessionID] = newData + + # return intervalSessions + newSessions = [] + + # Generate all edges tied to a user + # [ edge list, edge list, ... ] + for k, v in intervalSessions.items(): + pairs = pairwise(v) # list of edges for a user + newSessions.append(pairs) + + # Node Map + node_list = [] # Need to keep 0-based index for sankey diagram + links = [] # Aggregate sequence list + node_map = [] # Final node map {"name": "foo", "id": 0"} + + # Align the sequences + alignment = itertools.izip_longest(*newSessions) + src_ids = {} + target_ids = {} + + for i, step in enumerate(alignment): + # print(i) + c = collections.Counter() + visitedLinks = [] + # visitedLinksUnique = set([]) + nodenames = set([]) + + for edge in step: # for a single step look at all links + if edge: + node1 = edge[0] + node2 = edge[1] + session = node1['sessionID'] + nodename1 = node1['target'] + nodename2 = node2['target'] + + seqID = '%s->%s' % (nodename1, nodename2) + #print(seqID) + + if nodename1 != nodename2: #double check again for self-loops + #print(node1) + link = { + 'sequenceID': seqID, + 'sourceName': nodename1, + 'targetName': nodename2, + 'type': node1['type'], + 'duration': node1['duration'], + 'pathLength': len(node1['path']) if node1['path'] is not None else 0, + 'targetChange': node1['targetChange'], + 'typeChange': node1['typeChange'] + } + visitedLinks.append(link) + + # Done with visits in a step. Now calculate counts + counts = collections.Counter(k['sequenceID'] for k in visitedLinks if k.get('sequenceID')) + # print(counts) + visitedLinksUnique = { v['sequenceID']:v for v in visitedLinks}.values() + # print(visitedLinksUnique) + + # Visit unique links and generate src/targetid + if len(node_map) == 0: + for link in visitedLinksUnique: + # Add all sources + if link['sourceName'] not in src_ids: + node_map.append({"name": link['sourceName']}) + src_ids[link['sourceName']] = len(node_map)-1 + + # Add all targets + if link['targetName'] not in target_ids: + node_map.append({"name": link['targetName']}) + target_ids[link['targetName']] = len(node_map)-1 + + else: + src_ids = target_ids # sources were previous targets + target_ids = {} + for link in visitedLinksUnique: + # Add all sources + # if link['sourceName'] not in src_ids.values(): + # node_map.append(link['sourceName']) + # src_ids[len(node_map)-1] = link['sourceName'] + + # Add all targets + if link['targetName'] not in target_ids: + node_map.append({"name": link['targetName']}) + target_ids[link['targetName']] = len(node_map)-1 + + for link in visitedLinksUnique: + # Perform lookup for ids + # Perform lookup for counts + link['source'] = src_ids[link['sourceName']] + link['target'] = target_ids[link['targetName']] + link['value'] = counts[link['sequenceID']] + + links.append(link) + + # for step in alignment: + # # step through every users sequence + # c = collections.Counter() + # visitedLinks = [] + # nodenames = set([]) + # + # # Process all the edges + # for edge in step: + # if edge: + # node1 = edge[0] + # node2 = edge[1] + # + # nodename1 = node1['target'] + # nodename2 = node2['target'] + # + # # Add src and targetids + # nodenames.add(nodename1) + # nodenames.add(nodename2) + # + # # Generate sequence ID + # seqID = '%s->%s' % (nodename1, nodename2) + # + # # @todo Ensure src and target are not the same (self-loop) + # if nodename1 != nodename2: + # link = { + # 'sequenceID': seqID, + # 'sourceName': nodename1, + # 'targetName': nodename2, + # 'type': node1['type'], + # # 'duration': node1['duration'], + # 'pathLength': len(node1['path']), + # 'targetChange': node1['targetChange'], + # 'typeChange': node1['typeChange'] + # } + # visitedLinks.append(link) + # + # # How many users visited a sequence at this step + # counts = collections.Counter(k['sequenceID'] for k in visitedLinks if k.get('sequenceID')) + # # print(counts) + # # Append into growing node_list + # map(lambda x: node_list.append(x), nodenames) + # + # # map(lambda x: node_map.append({ "name": x} + # # "id": len(node_list) - 1 - node_list[::-1].index(x)}), nodenames) + # + # map(lambda x: node_map.append({ "name": x}), nodenames) + # # "id": len(node_list) - 1 - node_list[::-1].index(x)}), nodenames) + # for v in visitedLinks: + # # Pass through and update count, also generate src and target id + # v['value'] = counts[v['sequenceID']] + # # Last occurence is the src and target id + # v['source'] = len(node_list) -1 - node_list[::-1].index(v['sourceName']) + # v['target'] = len(node_list) -1 - node_list[::-1].index(v['targetName']) + # links.append(v) + + # Save everything + res = dict() + res['histogram'] = generate_bargraph(allTargets) + # res['sankey'] = { + # # 'sessions': sessions, + # 'links': links, + # 'nodes': node_map + # } + + res['nodes'] = node_map + res['links'] = links + res['sessions'] = sessions + # with open('sankey.json', 'w') as outfile: + # json.dump(res, outfile, sort_keys=False, indent=4) + + # with open('data.txt', 'w') as outfile: + # json.dump(intervalSessions, outfile, indent=4, sort_keys=False) + # + # with open('query.json', 'w') as outfile: + # json.dump(query, outfile, indent=4, sort_keys=False) + # Iterate first to get nodes + # pairs = pairwise(iter) + # + # nodes = [] + # links = [] + + # for p in pairs: + # node1 = p[0]['_source'] + # node2 = p[1]['_source'] + + # # Append nodes to list + # nodes.append(node1['target']) + # nodes.append(node2['target']) + + # Iterate again to get edges + # pairs = pairwise(iter2) + + # srcID = targetID = None + # for p in pairs: + # node1 = p[0]['_source'] + # node2 = p[1]['_source'] + # + # # Append nodes to list + # nodes.append(node1['target']) + # # nodes.append(node2['target']) + # + # srcID = len(nodes) - 1 + # targetID = len(nodes) + # + # # if (node1['target'] != node2['target']): + # # Append links to list (remove self-loops) + # link = { + # 'sourceID': srcID, + # 'targetID': targetID, + # 'sourceName': node1['target'], + # 'targetName': node2['target'], + # 'type': node1['type'], + # 'duration': node1['duration'], + # 'value': node1['count'], + # 'pathLength': len(node1['path']), + # 'targetChange': int(node1['targetChange']), + # 'typeChange': int(node1['typeChange']) + # } + # links.append(link) + # + # # Get all unique nodes + # # node_names = np.unique(nodes).tolist() + # node_list = [] + # + # for indx, name in enumerate(nodes): + # n = {'id': indx, 'name': name} + # node_list.append(n) + # + # # Remove self-loops + # newLinks = [] + # for indx, elem in enumerate(links): + # srcID = elem['sourceID'] + # targetID = elem['targetID'] + # + # if srcID != targetID: + # newLinks.append(elem) + # + + # + return res + + +def pairwise(iterable): + "s -> (s0, s1), (s1, s2), (s2, s3), ..." + a, b = itertools.tee(iterable) + next(b, None) + return itertools.izip(a, b) + + +def generate_bargraph(data, filename='bargraph.json'): + results = [] + for target in data: + target_name = target['key'] + types = [] + counts = [] + type_bucket = target['events']['buckets'] + for t in type_bucket: + types.append(t['key']) + counts.append(t['doc_count']) + + top_bucket = target['top_target']['hits']['hits'][0] + path_length = top_bucket['fields']['path_length'][0] + res = { + "target": target_name, + "counts": counts, + "types": types, + "pathLength": path_length + } + results.append(res) + + return results + + # with open(filename, 'w') as outfile: + # json.dump(results, outfile, indent=4, sort_keys=False) + +def generate_sankey(data, filename='sankey.json'): + pass diff --git a/WIP-distill_package/flagon-distill/version.py b/WIP-distill_package/flagon-distill/version.py new file mode 100644 index 0000000..d63fbb7 --- /dev/null +++ b/WIP-distill_package/flagon-distill/version.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Version information for Distill. + +This file is imported by ``Distill.__init__``, +and parsed by ``setup.py``. +""" + +__version__ = "0.1.0" diff --git a/WIP-distill_package/setup.cfg b/WIP-distill_package/setup.cfg new file mode 100644 index 0000000..e804958 --- /dev/null +++ b/WIP-distill_package/setup.cfg @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[egg_info] +tag_build = .dev +tag_date = 1 + +[aliases] +test=pytest +docs = build_sphinx +daily = egg_info --tag-date sdist +release = egg_info --tag-build=".rc" sdist bdist_wheel bdist_egg + +[tool:pytest] + +[build_sphinx] +source-dir = docs/source +build-dir = docs/_build +all_files = 1 + +[upload_sphinx] +upload-dir = docs/_build/html + +[bdist_wheel] +universal = 1 \ No newline at end of file diff --git a/WIP-distill_package/setup.py b/WIP-distill_package/setup.py new file mode 100644 index 0000000..5b1ca6a --- /dev/null +++ b/WIP-distill_package/setup.py @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools +import io +import os +import sys + +if sys.version_info[:2] < (2, 7): + m = "Python 2.7 or later is required for Distill (%d.%d detected)." + raise ImportError(m % sys.version_info[:2]) + +if sys.argv[-1] == 'setup.py': + #print("To install, run 'python setup.py install'\n") + #print("To run tests, run 'python setup.py test'\n") + #print("To build docs, run 'python setup.py docs'\n") + #print("To clean, run 'python setup.py clean'\n") + + +class CleanCommand(setuptools.Command): + """ + A command class to clean up build artifacts, including log files + + python setup.py clean + """ + description = "clean build artifacts" + user_options = [] + + def initialize_options(self): + """ + Initialize Clean Command + """ + self.cwd = None + + def finalize_options(self): + """ + Set Current Working directory for Clean Command + + """ + self.cwd = os.getcwd() + + def run(self): + """ + Execute Clean Command + """ + assert os.getcwd() == self.cwd, 'Must be in package root: %s' % self.cwd + os.system('rm -rf ./docs/_build ./build ./dist ./*.egg-info') + os.system('rm -rf ./*.log *.xml .coverage *.html') + +def read(*filenames, **kwargs): + encoding = kwargs.get('encoding', 'utf-8') + sep = kwargs.get('sep', '\n') + buf = [] + for filename in filenames: + with io.open(filename, encoding=encoding) as f: + buf.append(f.read()) + return sep.join(buf) + + +# Get the version string +def get_version(): + basedir = os.path.dirname(__file__) + with open(os.path.join(basedir, 'distill/version.py')) as f: + version = {} + exec (f.read(), version) + return version['__version__'] + raise RuntimeError('No version info found.') + + +setup_requires = [ + 'pip >= 9.0.1', + 'setuptools >= 34.0', + 'pytest-runner', +] + +install_requires = [ + 'elasticsearch-dsl >= 5.0.0, < 6.0.0', + 'pandas >= 0.20.2', + #'Flask >= 0.12.2', + #'celery >= 4.0.2', +] + +tests_require = [ + #'pytest >= 3.0.0', + #'pytest-pylint', + #'pytest-html', + #'pytest-cov', +] + +docs_require = [ + #'Sphinx >= 1.5.2', + #'sphinx-rtd-theme >= 0.1.9', +] + +extras_require = { + #'test' : tests_require, + #'doc' : docs_require, + #'monitor': [ + # 'flower' + #] +} + +classifiers = [ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research/Development', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + #'Programming Language :: Python :: 2.7', + #'Programming Language :: Python :: 3.5', + 'Environment :: Web Environment', + #'Framework :: Flask', + #'Topic :: Internet :: Log Analysis ' +] + +setuptools.setup( + name="Distill", + version=get_version(), + author="Joshua Poore, Michelle Beard", + author_email="[email protected]", + description="An analytical framework for UserALE.js", + long_description=__doc__, + long_description_content_type="text/markdown" + license="Apache Software License 2.0", + url="https://github.com/apache/incubator-flagon-distill";, + packages=setuptools.find_packages(exclude=['examples', 'tests']), + classifiers=classifiers, + keywords="userale distill", + include_package_data=True, + #zip_safe=False, + cmdclass={ + 'clean': CleanCommand, + }, + setup_requires=setup_requires, + tests_require=tests_require, + install_requires=install_requires, + extras_require=extras_require, + entry_points={ + 'console_scripts': [ + 'dev = distill.server:dev_server' + ] + } +)
