Hi CephFolks,
I have a cluster 14.2.21-22/Ubuntu 18.04 with 3 mon’s. After going down/restart
of 1 mon(amon3) it stucks on probing and its also out of quorum, we have
changed nothing and it was working, regardless we have checked tcp ports /
mtu's of mons are open and reachable.
appreciate any help you can provide.
ceph health detail
HEALTH_WARN 1/3 mons down, quorum amon1,amon2
MON_DOWN 1/3 mons down, quorum amon1,amon2
mon.amon3 (rank 0) addr [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] is
down (out of quorum)
all mon daemons are up and running, monmaptool printout is also equal on all
mon nodes and on other healthy mon nodes log there are entries from amon3:
Log from healthy amon1 log:
2021-09-10 10:49:16.657 7fc8285f3700 0 log_channel(audit) log [DBG] :
from='client.? 172.16.12.83:0/245515484' entity='client.admin' cmd=[{"prefix":
"status"}]: dispatch
2021-09-10 11:04:16.586 7fc8285f3700 0 log_channel(audit) log [DBG] :
from='client.? 172.16.12.83:0/2472330429' entity='client.admin' cmd=[{"prefix":
"status"}]: dispatch
2021-09-10 11:06:06.245 7fc8285f3700 0 log_channel(audit) log [DBG] :
from='client.? 172.16.12.83:0/1444461156' entity='client.admin' cmd=[{"prefix":
"mon getmap"}]: dispatch
2021-09-10 11:19:16.581 7fc8285f3700 0 log_channel(audit) log [DBG] :
from='client.? 172.16.12.83:0/964654151' entity='client.admin' cmd=[{"prefix":
"status"}]: dispatch
#########
monmaptool --print /tmp/monmap
monmaptool: monmap file /tmp/monmap
epoch 14
fsid 10a40588-d89e-4fac-be88-afcedc1a7372
last_changed 2021-06-22 13:35:56.211202
created 2018-07-10 12:37:03.124657
min_mon_release 14 (nautilus)
0: [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] mon.amon3
1: [v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] mon.amon1
2: [v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] mon.amon2
##########
ceph config get mon.*
WHO MASK LEVEL OPTION VALUE RO
mon advanced mon_crush_min_required_version firefly *
mon advanced mon_warn_on_insecure_global_id_reclaim_allowed false
##########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon3.asok mon_status
{
"name": "amon3",
"rank": 0,
"state": "probing",
"election_epoch": 0,
"quorum": [],
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "0",
"quorum_mon": []
},
"outside_quorum": [
"amon3"
],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 13:35:56.211202",
"created": "2018-07-10 12:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
]
}
}
#########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon2.asok mon_status
{
"name": "amon2",
"rank": 2,
"state": "peon",
"election_epoch": 4612,
"quorum": [
1,
2
],
"quorum_age": 65650,
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "4611087854035861503",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
]
},
"outside_quorum": [],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 11:35:56.211202",
"created": "2018-07-10 10:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
],
"osd": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 18
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 11
}
],
"client": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 30
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 7
}
],
"mgr": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 2
}
]
}
}
###########
ceph --admin-daemon /var/run/ceph/ceph-mon.amon1.asok mon_status
{
"name": "amon1",
"rank": 1,
"state": "leader",
"election_epoch": 4612,
"quorum": [
1,
2
],
"quorum_age": 65744,
"features": {
"required_con": "2449958747315912708",
"required_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"quorum_con": "4611087854035861503",
"quorum_mon": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
]
},
"outside_quorum": [],
"extra_probe_peers": [
{
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
}
],
"sync_provider": [],
"monmap": {
"epoch": 14,
"fsid": "10a40588-d89e-4fac-be88-afcedc1a7372",
"modified": "2021-06-22 13:35:56.211202",
"created": "2018-07-10 12:37:03.124657",
"min_mon_release": 14,
"min_mon_release_name": "nautilus",
"features": {
"persistent": [
"kraken",
"luminous",
"mimic",
"osdmap-prune",
"nautilus"
],
"optional": []
},
"mons": [
{
"rank": 0,
"name": "amon3",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.83:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.83:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.83:6789/0",
"public_addr": "172.16.12.83:6789/0"
},
{
"rank": 1,
"name": "amon1",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.81:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.81:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.81:6789/0",
"public_addr": "172.16.12.81:6789/0"
},
{
"rank": 2,
"name": "amon2",
"public_addrs": {
"addrvec": [
{
"type": "v2",
"addr": "172.16.12.82:3300",
"nonce": 0
},
{
"type": "v1",
"addr": "172.16.12.82:6789",
"nonce": 0
}
]
},
"addr": "172.16.12.82:6789/0",
"public_addr": "172.16.12.82:6789/0"
}
]
},
"feature_map": {
"mon": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
],
"osd": [
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 39
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 8
}
],
"client": [
{
"features": "0x3ffddff8ffacfffb",
"release": "luminous",
"num": 1
},
{
"features": "0x3ffddff8ffacffff",
"release": "luminous",
"num": 32
},
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 12
}
],
"mgr": [
{
"features": "0x3ffddff8ffecffff",
"release": "luminous",
"num": 1
}
]
}
}
########
Amon3 log
2021-09-10 12:06:16.409 7f1ead2033c0 0 starting mon.amon3 rank 0 at public
addrs [v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] at bind addrs
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] mon_data
/var/lib/ceph/mon/ceph-amon3 fsid 10a40588-d89e-4fac-be88-afcedc1a7372
2021-09-10 12:06:16.413 7f1ead2033c0 1 mon.amon3@-1(???) e14 preinit fsid
10a40588-d89e-4fac-be88-afcedc1a7372
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 check_fsid
cluster_uuid contains '10a40588-d89e-4fac-be88-afcedc1a7372'
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 features
compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos
with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap
encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support
monmap features,9=luminous ondisk layout,10=mimic ondisk layout,11=nautilus
ondisk layout}
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14
calc_quorum_requirements required_features 2449958747315912708
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 required_features
2449958747315912708
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 has_ever_joined =
1
2021-09-10 12:06:16.413 7f1ead2033c0 1 mon.amon3@-1(???) e14 preinit clean up
potentially inconsistent store state
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14
sync_last_committed_floor 353360965
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 init_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 5 mon.amon3@-1(???).mds e0 Unable to load
'last_metadata'
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).health init
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).config init
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???) e14 refresh_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).log v0
update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).log v0
update_from_paxos version 0 summary v 0
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).auth v0
update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).config load_config
got 0 keys
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).mgrstat 0
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).mgrstat check_subs
2021-09-10 12:06:16.413 7f1ead2033c0 10 mon.amon3@-1(???).health
update_from_paxos
2021-09-10 12:06:16.413 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] learned_addr learned my addr
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] (peer_addr_for_me
v2:172.16.12.83:3300/0)
2021-09-10 12:06:16.413 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] _finish_bind bind my_addrs is
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]
2021-09-10 12:06:16.413 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] start start
2021-09-10 12:06:16.413 7f1ead2033c0 1 -- start start
2021-09-10 12:06:16.413 7f1ead2033c0 2 mon.amon3@-1(???) e14 init
2021-09-10 12:06:16.417 7f1ead2033c0 1 Processor -- start
2021-09-10 12:06:16.417 7f1ead2033c0 1 Processor -- start
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(???).mgr e0
prime_mgr_client
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 bootstrap
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14
sync_reset_requester
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14
unregister_cluster_logger - not registered
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@-1(probing) e14 monmap e14: 3
mons at
{amon1=[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0],amon2=[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0],amon3=[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]}
2021-09-10 12:06:16.417 7f1ead2033c0 0 mon.amon3@-1(probing) e14 my rank is
now 0 (was -1)
2021-09-10 12:06:16.417 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] shutdown_connections
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 _reset
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing).auth v0
_set_mon_num_rank num 0 rank 0
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14
timecheck_finish
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14
scrub_event_cancel
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 scrub_reset
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14
reset_probe_timeout 0x55ffa1907860 after 2 seconds
2021-09-10 12:06:16.417 7f1ead2033c0 10 mon.amon3@0(probing) e14 probing other
monitors
2021-09-10 12:06:16.417 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] send_to--> mon
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] -- mon_probe(probe
10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- ?+0
0x55ffa0956a00
2021-09-10 12:06:16.417 7f1ead2033c0 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect
2021-09-10 12:06:16.417 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] -->
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] -- mon_probe(probe
10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 --
0x55ffa0956a00 con 0x55ffa096f600
2021-09-10 12:06:16.417 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] send_to--> mon
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] -- mon_probe(probe
10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 -- ?+0
0x55ffa0956780
2021-09-10 12:06:16.417 7f1ead2033c0 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect
2021-09-10 12:06:16.417 7f1ead2033c0 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] -->
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] -- mon_probe(probe
10a40588-d89e-4fac-be88-afcedc1a7372 name amon3 mon_release 14) v7 --
0x55ffa0956780 con 0x55ffa096fa80
2021-09-10 12:06:16.417 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.417 7f1e99e04700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:16.417 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:16.417 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:16.417 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:16.417 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.200000
2021-09-10 12:06:16.421 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.421 7f1e99603700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:16.425 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 37
2021-09-10 12:06:16.425 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:16.425 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:16.425 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.200000
2021-09-10 12:06:16.617 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.617 7f1e99e04700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:16.621 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:16.621 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:16.621 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:16.621 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.400000
2021-09-10 12:06:16.625 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:16.625 7f1e99603700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:16.625 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:16.625 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:16.625 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:16.625 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.400000
2021-09-10 12:06:17.021 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.021 7f1e99e04700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:17.021 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:17.021 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:17.021 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:17.021 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.800000
2021-09-10 12:06:17.029 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.029 7f1e99603700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:17.029 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:17.029 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:17.029 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:17.029 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 0.800000
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c000
0x55ffa1944000 :6789 s=ACCEPTING pgs=0 cs=0 l=0).send_server_banner sd=34
legacy v1:172.16.12.83:6789/0 socket_addr v1:172.16.12.83:6789/0 target_addr
v1:172.16.12.19:41104/0
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14
ms_handle_accept con 0x55ffa193c000 no session
2021-09-10 12:06:17.613 7f1e955fb700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] <== client.?
v1:172.16.12.19:0/10663437 1 ==== auth(proto 0 30 bytes epoch 0) v1 ==== 60+0+0
(unknown 900162395 0 0) 0x55ffa194c000 con 0x55ffa193c000
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 _ms_dispatch
new session 0x55ffa194c240 MonSession(client.? v1:172.16.12.19:0/10663437 is
open , features 0x3ffddff8ffecffff (luminous)) features 0x3ffddff8ffecffff
2021-09-10 12:06:17.613 7f1e955fb700 5 mon.amon3@0(probing) e14 waitlisting
message auth(proto 0 30 bytes epoch 0) v1
2021-09-10 12:06:17.613 7f1e94dfa700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437
conn(0x55ffa193c000 legacy=0x55ffa1944000 unknown :6789
s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 34
2021-09-10 12:06:17.613 7f1e94dfa700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437
conn(0x55ffa193c000 legacy=0x55ffa1944000 unknown :6789
s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437
conn(0x55ffa193c000 0x55ffa1944000 :6789 s=OPENED pgs=2 cs=1
l=1).handle_message read tag failed
2021-09-10 12:06:17.613 7f1e94dfa700 1 --1-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> v1:172.16.12.19:0/10663437
conn(0x55ffa193c000 0x55ffa1944000 :6789 s=OPENED pgs=2 cs=1 l=1).fault on
lossy channel, failing
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14
ms_handle_reset 0x55ffa193c000 v1:172.16.12.19:0/10663437
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 reset/close on
session client.? v1:172.16.12.19:0/10663437
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14 remove_session
0x55ffa194c240 client.? v1:172.16.12.19:0/10663437 features 0x3ffddff8ffecffff
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480
0x55ffa15d2a00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).accept
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480
0x55ffa15d2a00 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rev1=0 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.613 7f1e94dfa700 10 mon.amon3@0(probing) e14
handle_auth_request con 0x55ffa193c480 (start) method 1 payload 22
2021-09-10 12:06:17.613 7f1e94dfa700 10 mon.amon3@0(probing) e14
handle_auth_request haven't formed initial quorum, EBUSY
2021-09-10 12:06:17.613 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c480
0x55ffa15d2a00 crc :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop
2021-09-10 12:06:17.613 7f1e955fb700 10 mon.amon3@0(probing) e14
ms_handle_reset 0x55ffa193c480 -
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900
0x55ffa15d3100 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).accept
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900
0x55ffa15d3100 unknown :-1 s=BANNER_ACCEPTING pgs=0 cs=0 l=0 rev1=0 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.621 7f1e94dfa700 10 mon.amon3@0(probing) e14
handle_auth_request con 0x55ffa193c900 (start) method 1 payload 22
2021-09-10 12:06:17.621 7f1e94dfa700 10 mon.amon3@0(probing) e14
handle_auth_request haven't formed initial quorum, EBUSY
2021-09-10 12:06:17.621 7f1e94dfa700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >> conn(0x55ffa193c900
0x55ffa15d3100 crc :-1 s=AUTH_ACCEPTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop
2021-09-10 12:06:17.621 7f1e955fb700 10 mon.amon3@0(probing) e14
ms_handle_reset 0x55ffa193c900 -
2021-09-10 12:06:17.825 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.825 7f1e99e04700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:17.825 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:17.825 7f1e99e04700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
msgr2=0x55ffa15d1c00 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:17.825 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:17.825 7f1e99e04700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0] conn(0x55ffa096f600
0x55ffa15d1c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 1.600000
2021-09-10 12:06:17.829 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._handle_peer_banner_payload supported=1 required=0
2021-09-10 12:06:17.829 7f1e99603700 10 mon.amon3@0(probing) e14
ms_get_authorizer for mon
2021-09-10 12:06:17.829 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_bulk peer
close file descriptor 34
2021-09-10 12:06:17.829 7f1e99603700 1 --
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
msgr2=0x55ffa15d2300 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=0).read_until
read failed
2021-09-10 12:06:17.829 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 crc :-1 s=SESSION_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1)
Operation not permitted)
2021-09-10 12:06:17.829 7f1e99603700 1 --2-
[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0] >>
[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0] conn(0x55ffa096fa80
0x55ffa15d2300 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=0 rev1=1 rx=0
tx=0)._fault waiting 1.600000
2021-09-10 12:06:18.417 7f1e97e00700 4 mon.amon3@0(probing) e14 probe_timeout
0x55ffa1907860
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 bootstrap
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
sync_reset_requester
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
unregister_cluster_logger - not registered
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 monmap e14: 3
mons at
{amon1=[v2:172.16.12.81:3300/0,v1:172.16.12.81:6789/0],amon2=[v2:172.16.12.82:3300/0,v1:172.16.12.82:6789/0],amon3=[v2:172.16.12.83:3300/0,v1:172.16.12.83:6789/0]}
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 _reset
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing).auth v0
_set_mon_num_rank num 0 rank 0
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
timecheck_finish
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
scrub_event_cancel
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 scrub_reset
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
cancel_probe_timeout (none scheduled)
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14
reset_probe_timeout 0x55ffa1907890 after 2 seconds
2021-09-10 12:06:18.417 7f1e97e00700 10 mon.amon3@0(probing) e14 probing other
monitors
BR
Max
_______________________________________________
ceph-users mailing list -- [email protected]
To unsubscribe send an email to [email protected]
