This is an automated email from the ASF dual-hosted git repository. jky pushed a commit to branch kube in repository https://gitbox.apache.org/repos/asf/flagon.git
commit f9d670a0830a5e6e6e1589fc4185f6b7c108d7a4 Author: Jason Young <[email protected]> AuthorDate: Mon Jan 29 14:17:31 2024 -0500 Update kubernetes example --- kubernetes/6.2.2/elasticsearch/es-client-svc.yaml | 43 ---- kubernetes/6.2.2/elasticsearch/es-client.yaml | 78 ------ kubernetes/6.2.2/elasticsearch/es-data.yaml | 65 ----- kubernetes/6.2.2/elasticsearch/es-master-svc.yaml | 38 --- kubernetes/6.2.2/elasticsearch/es-master.yaml | 74 ------ kubernetes/6.2.2/kibana/kibana-svc.yaml | 15 -- kubernetes/6.2.2/kibana/kibana.yaml | 39 --- kubernetes/6.2.2/logstash/logstash-svc.yaml | 19 -- kubernetes/6.2.2/logstash/logstash.yaml | 52 ---- kubernetes/README.md | 294 +--------------------- kubernetes/configure.sh | 225 ----------------- kubernetes/resources/elastic.yaml | 22 ++ kubernetes/resources/logstash-auth.yaml | 11 + kubernetes/resources/logstash.yaml | 54 ++++ kubernetes/run.sh | 11 + kubernetes/test.py | 16 ++ 16 files changed, 119 insertions(+), 937 deletions(-) diff --git a/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml b/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml deleted file mode 100644 index 0d0622e..0000000 --- a/kubernetes/6.2.2/elasticsearch/es-client-svc.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Discovery Service for Elasticsearch client nodes over port 9200. -# -# A Kubernetes Service is an abstraction which defines a logical set of -# Pods and a policy by which to access them. -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch-loadbalancer - labels: - component: elk - role: client -spec: - selector: - component: elk - role: client - ports: - - name: http - port: 9200 -# targetPort: 9200 - protocol: TCP - type: NodePort -# type: Loadbalancer only works with certain providers: -# AWS, Azure, OpenStack, CloudStack, and Google Compute Engine -# type: LoadBalancer diff --git a/kubernetes/6.2.2/elasticsearch/es-client.yaml b/kubernetes/6.2.2/elasticsearch/es-client.yaml deleted file mode 100644 index 032ba18..0000000 --- a/kubernetes/6.2.2/elasticsearch/es-client.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-client - labels: - component: elk - role: client -spec: - replicas: 2 - template: - metadata: - labels: - component: elk - role: client - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-client - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Client node - - name: NODE_MASTER - value: "false" - - name: NODE_DATA - value: "false" - - name: NODE_INGEST - value: "false" - - name: HTTP_ENABLE - value: "true" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: "NETWORK_HOST" - value: _site_,_lo_ - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9200 - name: http - - containerPort: 9300 - name: transport - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /_cluster/health - port: http - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 10 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" diff --git a/kubernetes/6.2.2/elasticsearch/es-data.yaml b/kubernetes/6.2.2/elasticsearch/es-data.yaml deleted file mode 100644 index afe5617..0000000 --- a/kubernetes/6.2.2/elasticsearch/es-data.yaml +++ /dev/null @@ -1,65 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-data - labels: - component: elk - role: data -spec: - replicas: 2 - template: - metadata: - labels: - component: elk - role: data - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-data - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Data node - - name: NODE_MASTER - value: "false" - - name: NODE_DATA - value: "true" - - name: HTTP_ENABLE - value: "false" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9300 - name: transport - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" diff --git a/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml b/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml deleted file mode 100644 index 0cb1de8..0000000 --- a/kubernetes/6.2.2/elasticsearch/es-master-svc.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Discovery Service for Elasticsearch master nodes over port 9300. -# -# A Kubernetes Service is an abstraction which defines a logical set of -# Pods and a policy by which to access them. -apiVersion: v1 -kind: Service -metadata: - name: elasticsearch-discovery - labels: - component: elk - role: master -spec: - selector: - component: elk - role: master - ports: - - name: transport - port: 9300 - protocol: TCP diff --git a/kubernetes/6.2.2/elasticsearch/es-master.yaml b/kubernetes/6.2.2/elasticsearch/es-master.yaml deleted file mode 100644 index 592a00c..0000000 --- a/kubernetes/6.2.2/elasticsearch/es-master.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: es-master - labels: - component: elk - role: master -spec: - replicas: 3 - template: - metadata: - labels: - component: elk - role: master - spec: - initContainers: - - name: init-sysctl - image: busybox:1.27.2 - imagePullPolicy: IfNotPresent - command: - - sysctl - - -w - - vm.max_map_count=262144 - securityContext: - privileged: true - containers: - - name: es-master - image: senssoft/elasticsearch:6.2.2 - imagePullPolicy: IfNotPresent - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - # Dedicated master-eligible node - - name: NODE_MASTER - value: "true" - - name: NODE_DATA - value: "false" - - name: NODE_INGEST - value: "false" - - name: NODE_COORDINATE - value: "false" - - name: NUMBER_OF_MASTERS - value: "2" - - name: HTTP_ENABLE - value: "false" - - name: DISCOVERY_SERVICE - value: "elasticsearch-discovery" - - name: DISCOVERY_TIMEOUT - value: "30s" - - name: ES_JAVA_OPTS - value: -Xms256m -Xmx256m - ports: - - containerPort: 9300 - name: transport -# Need better readiness check policy than checking if port 9300 is active - livenessProbe: - tcpSocket: - port: transport - initialDelaySeconds: 20 - periodSeconds: 10 - failureThreshold: 5 - volumeMounts: - - name: storage - mountPath: /data - volumes: - - emptyDir: - medium: "" - name: "storage" diff --git a/kubernetes/6.2.2/kibana/kibana-svc.yaml b/kubernetes/6.2.2/kibana/kibana-svc.yaml deleted file mode 100644 index fbefde0..0000000 --- a/kubernetes/6.2.2/kibana/kibana-svc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kibana - labels: - component: elk - role: kibana -spec: - selector: - component: elk - role: kibana - ports: - - name: http - port: 5601 - type: NodePort \ No newline at end of file diff --git a/kubernetes/6.2.2/kibana/kibana.yaml b/kubernetes/6.2.2/kibana/kibana.yaml deleted file mode 100644 index f0f1ed0..0000000 --- a/kubernetes/6.2.2/kibana/kibana.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: kibana - labels: - component: elk - role: kibana -spec: - replicas: 1 - selector: - matchLabels: - component: elk - role: kibana - template: - metadata: - labels: - component: elk - role: kibana - spec: - containers: - - name: kibana - image: senssoft/kibana:6.2.2 - imagePullPolicy: IfNotPresent - resources: - # need more cpu upon initialization, therefore burstable class - limits: - cpu: 1000m - requests: - cpu: 100m - env: - - name: CLUSTER_NAME - value: SensSoft - - name: ELASTICSEARCH_URL - value: "http://elasticsearch-loadbalancer:9200"; - - name: ES_JAVA_OPTS - value: "-Xms512m -Xmx512m" - ports: - - containerPort: 5601 - name: http \ No newline at end of file diff --git a/kubernetes/6.2.2/logstash/logstash-svc.yaml b/kubernetes/6.2.2/logstash/logstash-svc.yaml deleted file mode 100644 index b77a032..0000000 --- a/kubernetes/6.2.2/logstash/logstash-svc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: logstash-discovery - labels: - component: elk - role: logstash -spec: - selector: - component: elk - role: logstash - ports: -# - name: transport -# port: 9600 -# protocol: TCP - - name: http - port: 8100 - protocol: TCP - type: NodePort diff --git a/kubernetes/6.2.2/logstash/logstash.yaml b/kubernetes/6.2.2/logstash/logstash.yaml deleted file mode 100644 index 8756731..0000000 --- a/kubernetes/6.2.2/logstash/logstash.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: logstash - labels: - component: elk - role: logstash -spec: - replicas: 1 - template: - metadata: - labels: - component: elk - role: logstash - spec: - containers: - - name: logstash - image: senssoft/logstash:6.2.2 - imagePullPolicy: IfNotPresent - command: - - logstash - - -f - - /etc/logstash/conf.d - env: - - name: ELASTICSEARCH_URL - value: "elasticsearch-loadbalancer:9200" - - name: ES_JAVA_OPTS - value: "-Xms1g -Xmx1g" - resources: - limits: - cpu: 1000m - requests: - cpu: 100m - ports: - - containerPort: 8100 - name: http - - containerPort: 9600 - name: transport - # Need better readiness check policy than checking if port 9600 is active -# livenessProbe: -# tcpSocket: -# port: transport -# initialDelaySeconds: 20 -# periodSeconds: 10 -# failureThreshold: 5 -# readinessProbe: -# httpGet: -# path: /?pretty=true -# port: http -# initialDelaySeconds: 30 -# periodSeconds: 10 -# failureThreshold: 10 \ No newline at end of file diff --git a/kubernetes/README.md b/kubernetes/README.md index b100fcc..35d50ee 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -1,293 +1,9 @@ -Apache SensSoft on top of Kubernetes -==================================== +# Example Kubernetes deployment -Apache SensSoft would like to thank [Pires] for his great documentation on deploying - the ELK stack in Kubernetes. -We have modified [Pires] instructions to fit Apache [SensSoft] requirements and guidelines. +This script and accompanying yaml files provide an example ELK stack kubernetes deployment. This is intended to be a starting point for deploying a userale logging end point. -Introduction ------------- +Prerequisites: A bash enviroment and Kubernetes running with at least 4 GB memory and 4 CPU cores. -The Software as a Sensor™ ([SensSoft]) Project offers an open-source (ALv2.0) software -tool usability testing platform. It includes a number of components that work together -to provide a platform for collecting data about user interactions with software tools, -as well as archiving, analyzing and visualizing that data. +Use the `run.sh` script to deploy the stack. -Apache SensSoft's logging infrastructure is powered by a family of [Elastic] tools, mainly -[Elasticsearch] for index and retrieval, [Logstash] for shipping data to Elasticsearch, -and [Kibana] for visualizing and building custom data-viz dashboards. - -This guide describes how to build and scale Elasticsearch clusters using Kubernetes. - -Current Elasticsearch version is `6.2.2`. - -Abstract --------- - -Before we start, one needs to know that Elasticsearch best-practices recommend to separate nodes in three roles: -* `Master` nodes - intended for clustering management only, no data, no HTTP API -* `Client` nodes - intended for client/search usage, no data, no HTTP API -* `Data` nodes - intended for storing and indexing your data, no HTTP API - -This is enforced throughout this document. - -Given this, I'm going to demonstrate how to provision a production grade scenario -consisting of 3 master, 2 client and 2 data nodes. - -(Very) Important Notes ----------------------- - -* Elasticsearch pods need for an init-container to run in privileged mode, so it -can set some VM options. For that to happen, the `kubelet` should be running with -args `--allow-privileged`, otherwise the init-container will fail to run. - -* By default, `ES_JAVA_OPTS` is set to `-Xms256m -Xmx256m`. This is a very low value - but many users, i.e. `minikube` users, were having issues with pods getting killed - because hosts were out of memory. One can change this in the deployment descriptors - available in this repository. - -* As of the moment, Kubernetes pod descriptors use an `emptyDir` for storing data -in each data node container. This is meant to be for the sake of simplicity and -should be adapted according to one's storage needs. - -Docker Images -------------- - -This example uses [this pre-built image](https://hub.docker.com/u/senssoft/) of the custom SensSoft ELK stack. - -Perquisites (Install and Setup) -------------------------------- - -Ensure that you have [Docker], [kubectl], and [minikube] (optional) installed and started. - -Deploy Elasticsearch Cluster -============================ - -Rollout Elasticsearch services and pod replicas. - -```bash -kubectl create -f elasticsearch/es-master-svc.yaml -kubectl create -f elasticsearch/es-client-svc.yaml -kubectl create -f elasticsearch/es-master.yaml -kubectl rollout status -f elasticsearch/es-master.yaml -kubectl create -f elasticsearch/es-client.yaml -kubectl rollout status -f elasticsearch/es-client.yaml -kubectl create -f elasticsearch/es-data.yaml -kubectl rollout status -f elasticsearch/es-data.yaml -``` - -or - -```bash -./configure deploy elasticsearch -``` - -Check one of the Elasticsearch master nodes logs: - -```bash -$ kubectl get svc,deployment,pods -l component=elk -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -svc/elasticsearch-discovery 10.103.242.73 <none> 9300/TCP 6m -svc/elasticsearch-loadbalancer 10.111.110.60 <nodes> 9200:30510/TCP 3m - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -deploy/es-client 2 2 2 2 3m -deploy/es-data 2 2 2 2 2m -deploy/es-master 3 3 3 3 6m - -NAME READY STATUS RESTARTS AGE -po/es-client-cbb74b6fb-tgbmp 1/1 Running 0 3m -po/es-client-cbb74b6fb-xnp5c 1/1 Running 0 3m -po/es-data-796d884bfb-fw8w5 1/1 Running 0 2m -po/es-data-796d884bfb-qs675 1/1 Running 0 2m -po/es-master-6f79799c8c-9sdtn 1/1 Running 0 6m -po/es-master-6f79799c8c-g6g9k 1/1 Running 0 6m -po/es-master-6f79799c8c-w2dcs 1/1 Running 0 6m -``` - -```bash -$ kubectl logs po/es-master-6f79799c8c-9sdtn -[2018-03-04T19:59:22,520][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] initializing ... -[2018-03-04T19:59:22,886][INFO ][o.e.e.NodeEnvironment ] [es-master-6f79799c8c-9sdtn] using [1] data paths, mounts [[/ (overlay)]], net usable_space [14gb], net total_space [16.1gb], types [overlay] -[2018-03-04T19:59:22,889][INFO ][o.e.e.NodeEnvironment ] [es-master-6f79799c8c-9sdtn] heap size [247.5mb], compressed ordinary object pointers [true] -[2018-03-04T19:59:22,902][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] node name [es-master-6f79799c8c-9sdtn], node ID [1720vLNASnmbPxwyElXeKQ] -[2018-03-04T19:59:22,904][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] version[6.2.2], pid[1], build[10b1edd/2018-02-16T19:01:30.685723Z], OS[Linux/4.9.64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_161/25.161-b14] -[2018-03-04T19:59:22,905][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, - [...] -[2018-03-04T19:59:27,519][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [aggs-matrix-stats] -[2018-03-04T19:59:27,530][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [analysis-common] -[2018-03-04T19:59:27,531][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [ingest-common] -[2018-03-04T19:59:27,533][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [lang-expression] -[2018-03-04T19:59:27,534][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [lang-mustache] -[2018-03-04T19:59:27,534][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [lang-painless] -[2018-03-04T19:59:27,534][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [mapper-extras] -[2018-03-04T19:59:27,540][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [parent-join] -[2018-03-04T19:59:27,551][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [percolator] -[2018-03-04T19:59:27,552][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [rank-eval] -[2018-03-04T19:59:27,552][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [reindex] -[2018-03-04T19:59:27,553][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [repository-url] -[2018-03-04T19:59:27,555][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [transport-netty4] -[2018-03-04T19:59:27,561][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded module [tribe] -[2018-03-04T19:59:27,565][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded plugin [ingest-geoip] -[2018-03-04T19:59:27,569][INFO ][o.e.p.PluginsService ] [es-master-6f79799c8c-9sdtn] loaded plugin [ingest-user-agent] -[2018-03-04T19:59:37,641][INFO ][o.e.d.DiscoveryModule ] [es-master-6f79799c8c-9sdtn] using discovery type [zen] -[2018-03-04T19:59:40,865][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] initialized -[2018-03-04T19:59:40,869][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] starting ... -[2018-03-04T19:59:42,001][INFO ][o.e.t.TransportService ] [es-master-6f79799c8c-9sdtn] publish_address {172.17.0.4:9300}, bound_addresses {0.0.0.0:9300} -[2018-03-04T19:59:42,079][INFO ][o.e.b.BootstrapChecks ] [es-master-6f79799c8c-9sdtn] bound or publishing to a non-loopback address, enforcing bootstrap checks -[2018-03-04T19:59:45,225][WARN ][o.e.d.z.ZenDiscovery ] [es-master-6f79799c8c-9sdtn] not enough master nodes discovered during pinging (found [[Candidate{node={es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300}, clusterStateVersion=-1}]], but needed [2]), pinging again -[2018-03-04T19:59:48,273][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-elected-as-master ([1] nodes joined)[{es-master-6f79799c8c-g6g9k}{9IQhhVe0Sj6reW9pPh0FsA}{FQzU-VNtRlexOnRaK9R2mQ}{172.17.0.6}{172.17.0.6:9300}], reason: new_master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300}, added {{es-master-6f79799c8c-g6g9k}{9IQhhVe0Sj6reW9pPh0FsA}{FQzU-VNtRlexOnRaK9R2mQ}{172.17.0.6}{172.17.0.6:9300},} -[2018-03-04T19:59:48,329][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] new_master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300}, added {{es-master-6f79799c8c-g6g9k}{9IQhhVe0Sj6reW9pPh0FsA}{FQzU-VNtRlexOnRaK9R2mQ}{172.17.0.6}{172.17.0.6:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed [...] -[2018-03-04T19:59:48,349][INFO ][o.e.n.Node ] [es-master-6f79799c8c-9sdtn] started -[2018-03-04T19:59:48,437][INFO ][o.e.g.GatewayService ] [es-master-6f79799c8c-9sdtn] recovered [0] indices into cluster_state -[2018-03-04T19:59:50,789][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-node-join[{es-master-6f79799c8c-w2dcs}{WAGDODrFRQOb_G9kh4MdBw}{vnfsSU2BQQKOYPB6S8tlHg}{172.17.0.5}{172.17.0.5:9300}], reason: added {{es-master-6f79799c8c-w2dcs}{WAGDODrFRQOb_G9kh4MdBw}{vnfsSU2BQQKOYPB6S8tlHg}{172.17.0.5}{172.17.0.5:9300},} -[2018-03-04T19:59:50,880][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] added {{es-master-6f79799c8c-w2dcs}{WAGDODrFRQOb_G9kh4MdBw}{vnfsSU2BQQKOYPB6S8tlHg}{172.17.0.5}{172.17.0.5:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed version [3] source [zen-disco-node-join[{es-master-6f79799c8c-w2dcs}{WAGDODrFRQOb_G9kh4MdBw}{vnfsSU2BQQKOYPB6S8tlHg}{1 [...] -[2018-03-04T20:01:07,315][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-node-join[{es-client-cbb74b6fb-tgbmp}{EunRsql8Q9CqSWJf0FKQbQ}{32GSGju1QbaYe30kTbG_5Q}{172.17.0.8}{172.17.0.8:9300}], reason: added {{es-client-cbb74b6fb-tgbmp}{EunRsql8Q9CqSWJf0FKQbQ}{32GSGju1QbaYe30kTbG_5Q}{172.17.0.8}{172.17.0.8:9300},} -[2018-03-04T20:01:07,751][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] added {{es-client-cbb74b6fb-tgbmp}{EunRsql8Q9CqSWJf0FKQbQ}{32GSGju1QbaYe30kTbG_5Q}{172.17.0.8}{172.17.0.8:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed version [4] source [zen-disco-node-join[{es-client-cbb74b6fb-tgbmp}{EunRsql8Q9CqSWJf0FKQbQ}{32GSGju1QbaYe30kTbG_5Q}{172 [...] -[2018-03-04T20:01:07,822][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-node-join[{es-client-cbb74b6fb-xnp5c}{SP4_qYhURHuQipeiiuC35w}{ISw5yvwPTUKMIFMvpMYXMA}{172.17.0.7}{172.17.0.7:9300}], reason: added {{es-client-cbb74b6fb-xnp5c}{SP4_qYhURHuQipeiiuC35w}{ISw5yvwPTUKMIFMvpMYXMA}{172.17.0.7}{172.17.0.7:9300},} -[2018-03-04T20:01:08,133][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] added {{es-client-cbb74b6fb-xnp5c}{SP4_qYhURHuQipeiiuC35w}{ISw5yvwPTUKMIFMvpMYXMA}{172.17.0.7}{172.17.0.7:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed version [5] source [zen-disco-node-join[{es-client-cbb74b6fb-xnp5c}{SP4_qYhURHuQipeiiuC35w}{ISw5yvwPTUKMIFMvpMYXMA}{172 [...] -[2018-03-04T20:02:16,606][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-node-join[{es-data-796d884bfb-qs675}{ZOopEPppSrGGk1f2uB6_Pg}{M7ZPAbbxQ0mnOpRZjNtq0A}{172.17.0.10}{172.17.0.10:9300}], reason: added {{es-data-796d884bfb-qs675}{ZOopEPppSrGGk1f2uB6_Pg}{M7ZPAbbxQ0mnOpRZjNtq0A}{172.17.0.10}{172.17.0.10:9300},} -[2018-03-04T20:02:17,310][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] added {{es-data-796d884bfb-qs675}{ZOopEPppSrGGk1f2uB6_Pg}{M7ZPAbbxQ0mnOpRZjNtq0A}{172.17.0.10}{172.17.0.10:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed version [6] source [zen-disco-node-join[{es-data-796d884bfb-qs675}{ZOopEPppSrGGk1f2uB6_Pg}{M7ZPAbbxQ0mnOpRZjNtq0A}{172 [...] -[2018-03-04T20:02:17,494][INFO ][o.e.c.s.MasterService ] [es-master-6f79799c8c-9sdtn] zen-disco-node-join[{es-data-796d884bfb-fw8w5}{GAMmdne1Q02pb0tujxGMTQ}{DsiN0UeYQQa9Gzeg_B3ARw}{172.17.0.9}{172.17.0.9:9300}], reason: added {{es-data-796d884bfb-fw8w5}{GAMmdne1Q02pb0tujxGMTQ}{DsiN0UeYQQa9Gzeg_B3ARw}{172.17.0.9}{172.17.0.9:9300},} -[2018-03-04T20:02:17,753][INFO ][o.e.c.s.ClusterApplierService] [es-master-6f79799c8c-9sdtn] added {{es-data-796d884bfb-fw8w5}{GAMmdne1Q02pb0tujxGMTQ}{DsiN0UeYQQa9Gzeg_B3ARw}{172.17.0.9}{172.17.0.9:9300},}, reason: apply cluster state (from master [master {es-master-6f79799c8c-9sdtn}{1720vLNASnmbPxwyElXeKQ}{S-_neLjjSPWj4ItZ1mv0zQ}{172.17.0.4}{172.17.0.4:9300} committed version [7] source [zen-disco-node-join[{es-data-796d884bfb-fw8w5}{GAMmdne1Q02pb0tujxGMTQ}{DsiN0UeYQQa9Gzeg_B3ARw}{172.1 [...] -``` - -As you can assert, the cluster is up and running. - -Access the service ------------------- - -Don't forget that services in Kubernetes are only accessible from containers in -the cluster. For different behavior one should configure the creation of an -external load-balancer. While it's supported within this example service -descriptor, its usage is out of scope of this document, for now. - -Note: if you are using one of the cloud providers which support external load -balancers, setting the type field to "LoadBalancer" will provision a load -balancer for your Service. You can uncomment the field in es-client-svc.yaml. - -```bash -$ kubectl get svc elasticsearch-loadbalancer -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -elasticsearch-loadbalancer 10.111.110.60 <nodes> 9200:30510/TCP 6m -``` - -From any host on your cluster (that's running `kube-proxy`), run: - -```bash -$ kubectl exec -it es-data-796d884bfb-fw8w5 -- /bin/bash -$ curl -XGET http://10.111.110.60:9200 -``` - -You should see something similar to the following: -```json -{ - "name" : "es-client-cbb74b6fb-tgbmp", - "cluster_name" : "SensSoft", - "cluster_uuid" : "J7-noNLVQP21dq8MZVnF-w", - "version" : { - "number" : "6.2.2", - "build_hash" : "10b1edd", - "build_date" : "2018-02-16T19:01:30.685723Z", - "build_snapshot" : false, - "lucene_version" : "7.2.1", - "minimum_wire_compatibility_version" : "5.6.0", - "minimum_index_compatibility_version" : "5.0.0" - }, - "tagline" : "You Know, for Search" -} -``` - -Or if you want to check cluster information: - -``` -curl -XGET http://10.111.110.60:9200/_cluster/health?pretty -``` - -You should see something similar to the following: - -```json -{ - "cluster_name" : "SensSoft", - "status" : "green", - "timed_out" : false, - "number_of_nodes" : 7, - "number_of_data_nodes" : 2, - "active_primary_shards" : 0, - "active_shards" : 0, - "relocating_shards" : 0, - "initializing_shards" : 0, - "unassigned_shards" : 0, - "delayed_unassigned_shards" : 0, - "number_of_pending_tasks" : 0, - "number_of_in_flight_fetch" : 0, - "task_max_waiting_in_queue_millis" : 0, - "active_shards_percent_as_number" : 100.0 -} -``` - -Deploy Logstash Service -======================= - -Rollout Logstash service - -```bash -$ ./configure deploy logstash -``` - -Check status -```bash -$ kubectl get svc,deployment,pods -l role=logstash -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -svc/logstash-discovery 10.103.234.11 <nodes> 8100:31010/TCP 3m - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -deploy/logstash 1 1 1 1 1m - -NAME READY STATUS RESTARTS AGE -po/logstash-9464b7fd8-dn8gv 1/1 Running 1 1m -``` - -Deploy Kibana Service -===================== - -Rollout Kibana service - -```bash -$ ./configure.sh deploy kibana -``` - -Check status -```bash -$ kubectl get svc,deployment,pods -l role=kibana -NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE -svc/kibana 10.103.73.161 <nodes> 5601:32422/TCP 1m - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -deploy/kibana 1 1 1 1 1m - -NAME READY STATUS RESTARTS AGE -po/kibana-b6f6b7b8f-plvkz 1/1 Running 0 1m -``` - -To view Kibana, either log into the VM or find your minikube VM address and -goto the assigned external port to view Kibana. - -```bash -$ firefox $(minikube ip):32422 -``` - -## Todo -1. Create new service `app` that will include pods tap, distill, site, db, etc. -2. Need a better way of checking on the state of Logstash. It needs the most time to provision. -3. Logstash is very unstable. - -[Pires]: https://github.com/pires/kubernetes-elasticsearch-cluster/ -[SensSoft]: http://senssoft.incubator.apache.org/ -[Elastic]: https://www.elastic.co/ -[Elasticsearch]: https://www.elastic.co/products/elasticsearch -[Logstash]: https://www.elastic.co/products/logstash -[Kibana]: https://www.elastic.co/products/kibana -[Cheat Sheet]: https://kubernetes.io/docs/user-guide/kubectl-cheatsheet/ -[Docker]: https://www.docker.com -[kubectl]: https://kubernetes.io/docs/tasks/tools/install-kubectl/ -[minikube]: https://kubernetes.io/docs/tasks/tools/install-minikube/ \ No newline at end of file +`test.py` is included as a utility to verify that logs are correctly posted to elastic locally. \ No newline at end of file diff --git a/kubernetes/configure.sh b/kubernetes/configure.sh deleted file mode 100755 index e6ae111..0000000 --- a/kubernetes/configure.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Print out usage documentation. -help_usage() { - echo "configure.sh" - echo "A simple utility to deploy Apache SensSoft Kubernetes build." - echo "Not meant to be used in production." - echo "" - echo "Usage: $ configure.sh COMMAND [OPT]" - echo "" - help_commands - echo "e.g." - echo "$ $0 deploy all" -} - -# Print out commands. -help_commands() { - echo "The commands are:" - echo " status View status of Kubernetes deployment" - echo " deploy elk Deploy ELK stack into Kubernetes cluster" - echo " deploy elasticsearch Deploy Elasticsearch Kubernetes" - echo " deploy logstash Deploy Logstash Kubernetes" - echo " deploy kibana Deploy Kibana Kubernetes" - echo " purge elk Purge all ELK Kubernetes artifacts" - echo " purge elasticsearch Purge all Elasticsearch Kubernetes artifacts" - echo " purge logstash Purge all Logstash Kubernetes artifacts" - echo " purge kibana Purge all Kibana Kubernetes artifacts" - echo " check Check environment for release" - echo " shutdown Shutdown Kubernetes cluster" - echo " start Startup minikube" - echo " stop Stop minikube" - echo " delete Delete minikube" - echo "" -} - -# If no arguments were provided, display the usage. -if [[ "$#" == "0" ]]; then - help_usage - exit 1 -fi - -# Check for a command argument. -COMMAND=$1 -COMMAND_OPT=$2 - -if [[ -z $COMMAND ]] || \ - [[ $COMMAND != "status" && \ - $COMMAND != "deploy" && \ - $COMMAND != "check" && \ - $COMMAND != "shutdown" && \ - $COMMAND != "purge" && \ - $COMMAND != "provision" && \ - $COMMAND != "stop" && \ - $COMMAND != "delete" ]]; then \ - echo "Error: Specify a command." - echo "" - help_commands - exit 1 -fi - -# Start minikube w/ hyperkit -if [[ $COMMAND == "provision" ]]; then - minikube start --cpus 2 --memory 5120 --vm-driver=virtualbox - # this for loop waits until kubectl can access the api server that Minikube has created - for i in {1..150}; do # timeout for 5 minutes - ./kubectl get po &> /dev/null - if [ $? -ne 1 ]; then - break - fi - sleep 2 - done - exit 0 -fi - -# Stop minikube w/ hyperkit -if [[ $COMMAND == "stop" ]]; then - minikube stop - eval $(minikube docker-env -u) - exit 0 -fi - -# Start minikube w/ hyperkit -if [[ $COMMAND == "delete" ]]; then - minikube delete - exit 0 -fi - -# Fetch status of entire Kubernets SensSoft namespace -if [[ $COMMAND == "status" ]]; then - kubectl get svc,deployment,pods -l component=elk - exit 0 -fi - -function elasticsearch() { - echo "Elasticsearch deployment" - kubectl create -f elasticsearch/es-master-svc.yaml - kubectl create -f elasticsearch/es-client-svc.yaml - kubectl create -f elasticsearch/es-master.yaml - kubectl rollout status -f elasticsearch/es-master.yaml - kubectl create -f elasticsearch/es-client.yaml - kubectl rollout status -f elasticsearch/es-client.yaml - kubectl create -f elasticsearch/es-data.yaml - kubectl rollout status -f elasticsearch/es-data.yaml -} - -function logstash() { - echo "Logstash deployment" - kubectl create -f logstash/logstash-svc.yaml - kubectl create -f logstash/logstash-client.yaml - kubectl create -f logstash/logstash.yaml - kubectl rollout status -f logstash/logstash.yaml -} - -function kibana() { - echo "Kibana deployment" - kubectl create -f kibana/kibana-svc.yaml - kubectl create -f kibana/kibana.yaml - kubectl rollout status -f kibana/kibana.yaml -} - -if [[ $COMMAND == "deploy" && \ - $COMMAND_OPT == "elasticsearch" ]]; then - elasticsearch - exit 0 -fi - -if [[ $COMMAND == "deploy" && \ - $COMMAND_OPT == "logstash" ]]; then - logstash - exit 0 -fi - -if [[ $COMMAND == "deploy" && \ - $COMMAND_OPT == "kibana" ]]; then - kibana - exit 0 -fi - -if [[ $COMMAND == "deploy" && \ - $COMMAND_OPT == "elk" ]]; then - elasticsearch - logstash - kibana - exit 0 -fi - -# Delete entire Elasticsearch Kubernetes artifacts -if [[ $COMMAND == 'purge' && \ - $COMMAND_OPT == 'elasticsearch' ]]; then - for f in elasticsearch/*.yaml - do - kubectl delete -f $f - done -fi - -# Delete entire Logstash Kubernetes artifacts -if [[ $COMMAND == 'purge' && \ - $COMMAND_OPT == 'logstash' ]]; then - for f in logstash/*.yaml - do - kubectl delete -f $f - done -fi - -# Delete entire Kibana Kubernetes artifacts -if [[ $COMMAND == 'purge' && \ - $COMMAND_OPT == 'kibana' ]]; then - for f in kibana/*.yaml - do - kubectl delete -f $f - done -fi - -# Delete entire Kubernetes SensSoft namespace -if [[ $COMMAND == 'purge' && \ - $COMMAND_OPT == 'elk' ]]; then - for f in elasticsearch/*.yaml - do - kubectl delete -f $f - done - for f in logstash/*.yaml - do - kubectl delete -f $f - done - for f in kibana/*.yaml - do - kubectl delete -f $f - done -fi - -# Prepare for Distill deployment -if [[ $COMMAND == "distill" ]]; then - echo "Error: Unsupported distill build." - exit 1 -fi - -# Prepare for Tap deployment -if [[ $COMMAND == "tap" ]]; then - echo "Error: Unsupported tap build." - exit 1 -fi - -# Run production build process checks. -if [[ $COMMAND == "check" ]]; then - echo "Error: Unsupported check build." - exit 1 -fi \ No newline at end of file diff --git a/kubernetes/resources/elastic.yaml b/kubernetes/resources/elastic.yaml new file mode 100644 index 0000000..ce201cd --- /dev/null +++ b/kubernetes/resources/elastic.yaml @@ -0,0 +1,22 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: quickstart +spec: + version: 8.12.0 + auth: + roles: + - secretName: my-roles-secret + nodeSets: + - name: default + count: 3 + podTemplate: + spec: + containers: + - name: elasticsearch + resources: + requests: + memory: 1Gi + cpu: 1 + limits: + memory: 1Gi \ No newline at end of file diff --git a/kubernetes/resources/logstash-auth.yaml b/kubernetes/resources/logstash-auth.yaml new file mode 100644 index 0000000..9935d5c --- /dev/null +++ b/kubernetes/resources/logstash-auth.yaml @@ -0,0 +1,11 @@ +kind: Secret +apiVersion: v1 +metadata: + name: my-roles-secret +stringData: + roles.yml: |- + eck_logstash_user_role: + cluster: [ "monitor", "manage_ilm", "read_ilm", "manage_logstash_pipelines", "manage_index_templates", "cluster:admin/ingest/pipeline/get"] + indices: + - names: [ "my-index", "logstash", "logstash-*", "ecs-logstash", "ecs-logstash-*", "logs-*", "metrics-*", "synthetics-*", "traces-*", "userale" ] + privileges: [ "manage", "write", "create_index", "read", "view_index_metadata" ] \ No newline at end of file diff --git a/kubernetes/resources/logstash.yaml b/kubernetes/resources/logstash.yaml new file mode 100644 index 0000000..f96e877 --- /dev/null +++ b/kubernetes/resources/logstash.yaml @@ -0,0 +1,54 @@ +apiVersion: logstash.k8s.elastic.co/v1alpha1 +kind: Logstash +metadata: + name: quickstart +spec: + count: 1 + elasticsearchRefs: + - name: quickstart + clusterName: qs + version: 8.12.0 + pipelines: + - pipeline.id: main + config.string: | + input { + http { + codec => "json" + port => 8100 + response_headers => { + "Access-Control-Allow-Origin" => "*" + "Access-Control-Allow-Headers" => "*" + } + } + } + + filter { + mutate { + remove_field => [ "message", "@version", "headers"] + } + + grok { + match => [ "useraleVersion", "(?<major_ver>\d+).(?<minor_ver>\d+)(.(?<patch_ver>\d+))?" ] + match => [ "toolVersion", "(?<tool_major_ver>\d+).(?<tool_minor_ver>\d+)(.(?<tool_patch_ver>\d+))?" ] + } + } + + output { + elasticsearch { + hosts => [ "${QS_ES_HOSTS}" ] + user => "${QS_ES_USER}" + password => "${QS_ES_PASSWORD}" + ssl_certificate_authorities => "${QS_ES_SSL_CERTIFICATE_AUTHORITY}" + index => "userale" + } + } + services: + - name: http + service: + spec: + type: NodePort + ports: + - port: 8100 + name: "http" + protocol: TCP + targetPort: 8100 \ No newline at end of file diff --git a/kubernetes/run.sh b/kubernetes/run.sh new file mode 100755 index 0000000..2051b00 --- /dev/null +++ b/kubernetes/run.sh @@ -0,0 +1,11 @@ +kubectl create -f https://download.elastic.co/downloads/eck/2.11.0/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/2.11.0/operator.yaml +kubectl apply -f resources/logstash-auth.yaml +kubectl apply -f resources/elastic.yaml +kubectl apply -f resources/logstash.yaml +sleep 10 +kubectl wait --for=condition=Ready pod/quickstart-es-default-0 --timeout=60s +kubectl wait --for=condition=Ready pod/quickstart-ls-0 --timeout=60s +kubectl port-forward service/quickstart-es-http 9200 >/dev/null 2>&1 & +kubectl port-forward service/quickstart-ls-http 8100 >/dev/null 2>&1 & +export ES_PASSWORD=$(kubectl get secret quickstart-es-elastic-user -o go-template='{{.data.elastic | base64decode}}') \ No newline at end of file diff --git a/kubernetes/test.py b/kubernetes/test.py new file mode 100644 index 0000000..cf5ddf5 --- /dev/null +++ b/kubernetes/test.py @@ -0,0 +1,16 @@ +from elasticsearch_dsl import connections +from elasticsearch_dsl import Search +from elasticsearch_dsl import Q +import os + +flagonClient = connections.create_connection('flagonTest', hosts=['https://localhost:9200'], basic_auth=("elastic", os.environ['ES_PASSWORD']), verify_certs=False) +AleS = Search(using='flagonTest', index="userale") +qLogType = Q("match", logType="raw") | Q("match", logType="custom") + +elk_search = AleS \ + .query(qLogType) \ + .extra(track_total_hits=True) #breaks return limit of 10000 hits + +elk_response = elk_search.scan() +for hit in elk_response: + print(hit) \ No newline at end of file
