BFergerson commented on issue #6234:
URL: https://github.com/apache/skywalking/issues/6234#issuecomment-763742943
@kezhenxu94, thanks for the help. That did not seem to solve the issue
though. It is now throwing `MasterNotDiscoveredException`. Again, I'm starting
from a fresh digital server (Ubuntu 20) and I've included the server's history
and logs below. Also, I've tried waiting 5-10 minutes for the issue to go away
and it doesn't so I don't think it's just me not waiting long enough.
---
### History
```
1 curl
https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
2 sudo snap install microk8s --classic
3 microk8s.status --wait-ready
4 microk8s.enable dns
5 snap alias microk8s.kubectl kubectl
6 microk8s.kubectl config view --raw > $HOME/.kube/config
7 git clone https://github.com/apache/skywalking-kubernetes
8 cd skywalking-kubernetes/chart
9 helm repo add elastic https://helm.elastic.co
10 helm dep up skywalking
11 export SKYWALKING_RELEASE_NAME=skywalking
12 export SKYWALKING_RELEASE_NAMESPACE=default
13 helm install "${SKYWALKING_RELEASE_NAME}" skywalking -n
"${SKYWALKING_RELEASE_NAMESPACE}" -f ./skywalking/values-es7.yaml
14 kubectl apply -f
https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0/aio/deploy/recommended.yaml
15 cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
EOF
16 cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
EOF
17 kubectl -n kubernetes-dashboard get secret $(kubectl -n
kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o
go-template="{{.data.token | base64decode}}"
18 kubectl proxy --address=0.0.0.0
```
### microk8s.status
```
microk8s is running
high-availability: no
datastore master nodes: 127.0.0.1:19001
datastore standby nodes: none
addons:
enabled:
dashboard # The Kubernetes dashboard
dns # CoreDNS
ha-cluster # Configure high availability on the current node
disabled:
ambassador # Ambassador API Gateway and Ingress
cilium # SDN, fast with full network policy
fluentd # Elasticsearch-Fluentd-Kibana logging and
monitoring
gpu # Automatic enablement of Nvidia CUDA
helm # Helm 2 - the package manager for Kubernetes
helm3 # Helm 3 - Kubernetes package manager
host-access # Allow Pods connecting to Host services smoothly
ingress # Ingress controller for external access
istio # Core Istio service mesh services
jaeger # Kubernetes Jaeger operator with its simple config
keda # Kubernetes-based Event Driven Autoscaling
knative # The Knative framework on Kubernetes.
kubeflow # Kubeflow for easy ML deployments
linkerd # Linkerd is a service mesh for Kubernetes and
other frameworks
metallb # Loadbalancer for your Kubernetes cluster
metrics-server # K8s Metrics Server for API access to service
metrics
multus # Multus CNI enables attaching multiple network
interfaces to pods
portainer # Portainer UI for your Kubernetes cluster
prometheus # Prometheus operator for monitoring and logging
rbac # Role-Based Access Control for authorisation
registry # Private image registry exposed on localhost:32000
storage # Storage class; allocates storage from host
directory
traefik # traefik Ingress controller for external access
```
### Logs
```
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated
in version 9.0 and will likely be removed in a future release.
{"type": "server", "timestamp": "2021-01-20T15:52:54,443Z", "level": "INFO",
"component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "using [1] data paths, mounts
[[/ (overlay)]], net usable_space [73.3gb], net total_space [77.3gb], types
[overlay]" }
{"type": "server", "timestamp": "2021-01-20T15:52:54,453Z", "level": "INFO",
"component": "o.e.e.NodeEnvironment", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "heap size [1015.6mb],
compressed ordinary object pointers [true]" }
{"type": "server", "timestamp": "2021-01-20T15:52:54,468Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "node name [elasticsearch-master-0], node
ID [98WRKYI_Tuq8O-vFLsRO2w], cluster name [elasticsearch]" }
{"type": "server", "timestamp": "2021-01-20T15:52:54,469Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "version[7.5.1], pid[1],
build[default/docker/3ae9ac9a93c95bd0cdc054951cf95d88e1e18d96/2019-12-16T22:57:37.835892Z],
OS[Linux/5.4.0-51-generic/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server
VM/13.0.1/13.0.1+9]" }
{"type": "server", "timestamp": "2021-01-20T15:52:54,476Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "JVM home [/usr/share/elasticsearch/jdk]" }
{"type": "server", "timestamp": "2021-01-20T15:52:54,477Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "JVM arguments
[-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10,
-XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8,
-Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true,
-Dio.netty.noKeySetOptimization=true,
-Dio.netty.recycler.maxCapacityPerThread=0,
-Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false,
-Dlog4j2.disable.jmx=true, -Djava.locale.providers=COMPAT, -Xms1g, -Xmx1g,
-XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75,
-XX:+UseCMSInitiatingOccupancyOnly,
-Djava.io.tmpdir=/tmp/elasticsearch-10059330462906371604,
-XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data,
-XX:ErrorFile=logs/hs_err_pid%p.log,
-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=3
2,filesize=64m, -Des.cgroups.hierarchy.override=/, -Xmx1g, -Xms1g,
-XX:MaxDirectMemorySize=536870912, -Des.path.home=/usr/share/elasticsearch,
-Des.path.conf=/usr/share/elasticsearch/config,
-Des.distribution.flavor=default, -Des.distribution.type=docker,
-Des.bundled_jdk=true]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,920Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[aggs-matrix-stats]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,921Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[analysis-common]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,922Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [flattened]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,923Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[frozen-indices]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,923Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[ingest-common]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,923Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[ingest-geoip]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,925Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[ingest-user-agent]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,926Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[lang-expression]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,926Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[lang-mustache]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,927Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[lang-painless]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,927Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[mapper-extras]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,927Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [parent-join]"
}
{"type": "server", "timestamp": "2021-01-20T15:52:59,928Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [percolator]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,929Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [rank-eval]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,929Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [reindex]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,930Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[repository-url]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,930Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[search-business-rules]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,931Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [spatial]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,932Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [transform]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,932Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[transport-netty4]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,934Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [vectors]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,935Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-analytics]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,935Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ccr]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,935Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-core]"
}
{"type": "server", "timestamp": "2021-01-20T15:52:59,936Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-deprecation]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,936Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-enrich]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,937Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-graph]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,937Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ilm]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,937Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-logstash]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,937Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-ml]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,938Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-monitoring]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,938Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-rollup]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,939Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-security]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,939Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module [x-pack-sql]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,939Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-voting-only-node]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,940Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "loaded module
[x-pack-watcher]" }
{"type": "server", "timestamp": "2021-01-20T15:52:59,940Z", "level": "INFO",
"component": "o.e.p.PluginsService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "no plugins loaded" }
{"type": "server", "timestamp": "2021-01-20T15:53:10,113Z", "level": "INFO",
"component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "parsed [0] roles from file
[/usr/share/elasticsearch/config/roles.yml]" }
{"type": "server", "timestamp": "2021-01-20T15:53:11,625Z", "level": "INFO",
"component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message":
"[controller/87] [Main.cc@110] controller (64 bit): Version 7.5.1 (Build
ae3c3c51b849be) Copyright (c) 2019 Elasticsearch BV" }
{"type": "server", "timestamp": "2021-01-20T15:53:12,911Z", "level":
"DEBUG", "component": "o.e.a.ActionModule", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "Using REST wrapper from
plugin org.elasticsearch.xpack.security.Security" }
{"type": "server", "timestamp": "2021-01-20T15:53:13,238Z", "level": "INFO",
"component": "o.e.d.DiscoveryModule", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "using discovery type [zen]
and seed hosts providers [settings]" }
{"type": "server", "timestamp": "2021-01-20T15:53:15,289Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "initialized" }
{"type": "server", "timestamp": "2021-01-20T15:53:15,290Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "starting ..." }
{"type": "server", "timestamp": "2021-01-20T15:53:15,590Z", "level": "INFO",
"component": "o.e.t.TransportService", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "publish_address
{10.1.137.6:9300}, bound_addresses {[::]:9300}" }
{"type": "server", "timestamp": "2021-01-20T15:53:15,904Z", "level": "INFO",
"component": "o.e.b.BootstrapChecks", "cluster.name": "elasticsearch",
"node.name": "elasticsearch-master-0", "message": "bound or publishing to a
non-loopback address, enforcing bootstrap checks" }
{"type": "server", "timestamp": "2021-01-20T15:53:25,926Z", "level": "WARN",
"component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not
discovered yet, this node has not previously joined a bootstrapped (v7+)
cluster, and this node must discover master-eligible nodes
[elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2] to
bootstrap a cluster: have discovered
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}]; discovery will continue using []
from hosts providers and
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state;
node term 0, last-accepted version 0 in term 0" }
{"type": "server", "timestamp": "2021-01-20T15:53:35,928Z", "level": "WARN",
"component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not
discovered yet, this node has not previously joined a bootstrapped (v7+)
cluster, and this node must discover master-eligible nodes
[elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2] to
bootstrap a cluster: have discovered
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}]; discovery will continue using []
from hosts providers and
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state;
node term 0, last-accepted version 0 in term 0" }
{"type": "server", "timestamp": "2021-01-20T15:53:45,930Z", "level": "WARN",
"component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not
discovered yet, this node has not previously joined a bootstrapped (v7+)
cluster, and this node must discover master-eligible nodes
[elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2] to
bootstrap a cluster: have discovered
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}]; discovery will continue using []
from hosts providers and
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state;
node term 0, last-accepted version 0 in term 0" }
{"type": "server", "timestamp": "2021-01-20T15:53:45,938Z", "level": "WARN",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "timed out while waiting for initial
discovery state - timeout: 30s" }
{"type": "server", "timestamp": "2021-01-20T15:53:45,948Z", "level": "INFO",
"component": "o.e.h.AbstractHttpServerTransport", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message":
"publish_address {10.1.137.6:9200}, bound_addresses {[::]:9200}" }
{"type": "server", "timestamp": "2021-01-20T15:53:45,949Z", "level": "INFO",
"component": "o.e.n.Node", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "started" }
{"type": "server", "timestamp": "2021-01-20T15:53:54,959Z", "level":
"DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "no known
master node, scheduling a retry" }
{"type": "server", "timestamp": "2021-01-20T15:53:55,932Z", "level": "WARN",
"component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "master not
discovered yet, this node has not previously joined a bootstrapped (v7+)
cluster, and this node must discover master-eligible nodes
[elasticsearch-master-0, elasticsearch-master-1, elasticsearch-master-2] to
bootstrap a cluster: have discovered
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}]; discovery will continue using []
from hosts providers and
[{elasticsearch-master-0}{98WRKYI_Tuq8O-vFLsRO2w}{ROMFxL8UT5egSM4p5bvGFw}{10.1.137.6}{10.1.137.6:9300}{dilm}{ml.machine_memory=2147483648,
xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state;
node term 0, last-accepted version 0 in term 0" }
{"type": "server", "timestamp": "2021-01-20T15:53:55,965Z", "level":
"DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out
while retrying [cluster:monitor/health] after failure (timeout [1s])" }
{"type": "server", "timestamp": "2021-01-20T15:53:55,967Z", "level": "WARN",
"component": "r.suppressed", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "path: /_cluster/health, params:
{wait_for_status=green, timeout=1s}",
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException:
null",
"at
org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:218)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[?:?]",
"at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[?:?]",
"at java.lang.Thread.run(Thread.java:830) [?:?]"] }
{"type": "server", "timestamp": "2021-01-20T15:54:04,853Z", "level":
"DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "no known
master node, scheduling a retry" }
{"type": "server", "timestamp": "2021-01-20T15:54:05,854Z", "level":
"DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name":
"elasticsearch", "node.name": "elasticsearch-master-0", "message": "timed out
while retrying [cluster:monitor/health] after failure (timeout [1s])" }
{"type": "server", "timestamp": "2021-01-20T15:54:05,855Z", "level": "WARN",
"component": "r.suppressed", "cluster.name": "elasticsearch", "node.name":
"elasticsearch-master-0", "message": "path: /_cluster/health, params:
{wait_for_status=green, timeout=1s}",
"stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException:
null",
"at
org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:218)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703)
[elasticsearch-7.5.1.jar:7.5.1]",
"at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
[?:?]",
"at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
[?:?]",
"at java.lang.Thread.run(Thread.java:830) [?:?]"] }
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
