Merge branch 'master' into pr-137
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/546139f5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/546139f5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/546139f5 Branch: refs/heads/master Commit: 546139f5ae8d4c11821dea7cae45462a61e2e81c Parents: 5957e22 bcda92e Author: Tadd Wood <t...@arcadiadata.com> Authored: Thu Sep 20 12:46:36 2018 -0700 Committer: Tadd Wood <t...@arcadiadata.com> Committed: Thu Sep 20 12:46:36 2018 -0700 ---------------------------------------------------------------------- README.md | 19 +- issues-subscr...@spot.incubator.apache.org | 0 spot-ingest/README.md | 6 + spot-ingest/common/configurator.py | 119 ++++++++++ spot-ingest/common/hdfs_client.py | 233 +++++++++++++++++++ spot-ingest/common/hive_engine.py | 73 ++++++ spot-ingest/common/kafka_client.py | 193 ++++++++++++---- spot-ingest/common/kerberos.py | 42 ++-- spot-ingest/kerberos-requirements.txt | 4 + spot-ingest/master_collector.py | 21 +- spot-ingest/pipelines/dns/collector.py | 133 +++++++---- spot-ingest/pipelines/dns/worker.py | 141 +++++++++--- spot-ingest/pipelines/flow/collector.py | 111 +++++---- spot-ingest/pipelines/flow/worker.py | 193 ++++++++++++---- spot-ingest/pipelines/proxy/bluecoat.py | 177 +++++++++----- spot-ingest/pipelines/proxy/collector.py | 6 +- spot-ingest/requirements.txt | 5 +- spot-ingest/start_ingest_standalone.sh | 2 +- spot-ingest/worker.py | 6 +- spot-oa/api/resources/configurator.py | 69 +++++- spot-oa/api/resources/hdfs_client.py | 201 ++++++++++++++-- spot-oa/api/resources/impala_engine.py | 29 ++- spot-oa/kerberos-requirements.txt | 3 + spot-oa/oa/components/data/hive.py | 1 + spot-oa/oa/components/data/impala.py | 25 +- spot-oa/oa/utils.py | 282 +++++++++++------------ spot-oa/requirements.txt | 2 + spot-setup/KERBEROS.md | 50 ++++ spot-setup/README.md | 7 + spot-setup/beeline/create_dns_parquet.hql | 162 +++++++++++++ spot-setup/beeline/create_flow_parquet.hql | 194 ++++++++++++++++ spot-setup/beeline/create_proxy_parquet.hql | 179 ++++++++++++++ spot-setup/create_dns_parquet.hql | 163 ------------- spot-setup/create_flow_parquet.hql | 195 ---------------- spot-setup/create_proxy_parquet.hql | 177 -------------- spot-setup/hdfs_setup.sh | 120 ++++++++-- spot-setup/hive/create_dns_parquet.hql | 165 +++++++++++++ spot-setup/hive/create_flow_parquet.hql | 197 ++++++++++++++++ spot-setup/hive/create_proxy_parquet.hql | 179 ++++++++++++++ spot-setup/impala/create_dns_parquet.hql | 163 +++++++++++++ spot-setup/impala/create_flow_parquet.hql | 195 ++++++++++++++++ spot-setup/impala/create_proxy_parquet.hql | 177 ++++++++++++++ spot-setup/spot.conf | 28 ++- 43 files changed, 3407 insertions(+), 1040 deletions(-) ----------------------------------------------------------------------
