[
https://issues.apache.org/jira/browse/HDDS-8441?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Arun Sarin updated HDDS-8441:
-----------------------------
Description:
Scenario Repro :
# Close container of a key
# Simulate one unhealthy replica in above container by deleting
hdds.datanode.dir
# New replica is added for the unhealthy replica but after successful addition
the unhealthy replica is not removed
Final state of container - 2 replicas on replica index 1
{code:java}
{
"containerInfo": {
"state": "CLOSED",
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"usedBytes": 54525952,
"numberOfKeys": 1,
"lastUsed": "2023-04-11T11:10:42.412Z",
"stateEnterTime": "2023-04-11T10:54:42.254Z",
"owner": "om3",
"containerID": 8001,
"deleteTransactionId": 0,
"sequenceId": 0,
"deleted": "False",
"open": "False"
},
"pipeline": {
"id": {
"id": "10064036-6251-40a9-b4fa-e1edeae388e5"
},
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"nodesInOrder": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"creationTimestamp": "2023-04-11T11:10:42.344Z",
"closed": "False",
"allocationTimeout": "False",
"healthy": "True",
"pipelineState": "ALLOCATED",
"nodeSet": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"leaderNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"firstNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"closestNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"nodes": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"empty": "False",
"type": "EC",
"open": "False"
},
"replicas": [
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"placeOfBirth": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 5
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
"placeOfBirth": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
"placeOfBirth": "74078315-7c10-412b-97a2-f520ad02b91e",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 4
},
{
"containerID": 8001,
"state": "UNHEALTHY",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
"placeOfBirth": "e6abd931-74f4-4a83-87ac-ad104f956706",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
"placeOfBirth": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 3
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
},
"placeOfBirth": "1548294e-6375-41e1-9324-0646a8163294",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 2
}
],
"Datanodes": [
"hostname-2.hostname.root.hwx.site",
"hostname-4.hostname.root.hwx.site",
"hostname-7.hostname.root.hwx.site",
"hostname-9.hostname.root.hwx.site",
"hostname-3.hostname.root.hwx.site",
"hostname-6.hostname.root.hwx.site"
]
} {code}
Error Message from TC:
{code:java}
replace_status = Ozone.check_replica_present_in_container(dn_i,
container_list[0], False)
> assert replace_status, "Unhealthy replica not replace by healthy replica"E
> AssertionError: Unhealthy replica not replace by healthy replica
E assert False{code}
DN didn't realised that the container is not there any more (eg the full volume
has failed)
was:
Scenario Repro :
# Close container of a key
# Simulate one unhealthy replica in above container by deleting
hdds.datanode.dir
# New replica is added for the unhealthy replica but after successful addition
the unhealthy replica is not removed
Final state of container - 2 replicas on replica index 1
{code:java}
{
"containerInfo": {
"state": "CLOSED",
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"usedBytes": 54525952,
"numberOfKeys": 1,
"lastUsed": "2023-04-11T11:10:42.412Z",
"stateEnterTime": "2023-04-11T10:54:42.254Z",
"owner": "om3",
"containerID": 8001,
"deleteTransactionId": 0,
"sequenceId": 0,
"deleted": "False",
"open": "False"
},
"pipeline": {
"id": {
"id": "10064036-6251-40a9-b4fa-e1edeae388e5"
},
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"nodesInOrder": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"creationTimestamp": "2023-04-11T11:10:42.344Z",
"closed": "False",
"allocationTimeout": "False",
"healthy": "True",
"pipelineState": "ALLOCATED",
"nodeSet": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"leaderNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"firstNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"closestNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"nodes": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"empty": "False",
"type": "EC",
"open": "False"
},
"replicas": [
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"placeOfBirth": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 5
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
"placeOfBirth": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
"placeOfBirth": "74078315-7c10-412b-97a2-f520ad02b91e",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 4
},
{
"containerID": 8001,
"state": "UNHEALTHY",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
"placeOfBirth": "e6abd931-74f4-4a83-87ac-ad104f956706",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
"placeOfBirth": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 3
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
},
"placeOfBirth": "1548294e-6375-41e1-9324-0646a8163294",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 2
}
],
"Datanodes": [
"hostname-2.hostname.root.hwx.site",
"hostname-4.hostname.root.hwx.site",
"hostname-7.hostname.root.hwx.site",
"hostname-9.hostname.root.hwx.site",
"hostname-3.hostname.root.hwx.site",
"hostname-6.hostname.root.hwx.site"
]
} {code}
Error Message from TC:
replace_status = Ozone.check_replica_present_in_container(dn_i,
container_list[0], False)
> assert replace_status, "Unhealthy replica not replace by healthy
> replica"E AssertionError: Unhealthy replica not replace by healthy
> replica
E assert False
DN didn't realised that the container is not there any more (eg the full volume
has failed)
> [EC] Unhealthy replica not getting replaced by healthy replica when
> hdds.datanode.dir path is deleted
> -----------------------------------------------------------------------------------------------------
>
> Key: HDDS-8441
> URL: https://issues.apache.org/jira/browse/HDDS-8441
> Project: Apache Ozone
> Issue Type: Bug
> Reporter: Arun Sarin
> Priority: Major
>
> Scenario Repro :
> # Close container of a key
> # Simulate one unhealthy replica in above container by deleting
> hdds.datanode.dir
> # New replica is added for the unhealthy replica but after successful
> addition the unhealthy replica is not removed
> Final state of container - 2 replicas on replica index 1
> {code:java}
> {
> "containerInfo": {
> "state": "CLOSED",
> "replicationConfig": {
> "data": 3,
> "parity": 2,
> "ecChunkSize": 1048576,
> "codec": "RS",
> "replicationType": "EC",
> "requiredNodes": 5
> },
> "usedBytes": 54525952,
> "numberOfKeys": 1,
> "lastUsed": "2023-04-11T11:10:42.412Z",
> "stateEnterTime": "2023-04-11T10:54:42.254Z",
> "owner": "om3",
> "containerID": 8001,
> "deleteTransactionId": 0,
> "sequenceId": 0,
> "deleted": "False",
> "open": "False"
> },
> "pipeline": {
> "id": {
> "id": "10064036-6251-40a9-b4fa-e1edeae388e5"
> },
> "replicationConfig": {
> "data": 3,
> "parity": 2,
> "ecChunkSize": 1048576,
> "codec": "RS",
> "replicationType": "EC",
> "requiredNodes": 5
> },
> "nodesInOrder": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "creationTimestamp": "2023-04-11T11:10:42.344Z",
> "closed": "False",
> "allocationTimeout": "False",
> "healthy": "True",
> "pipelineState": "ALLOCATED",
> "nodeSet": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "leaderNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "firstNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "closestNode": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "nodes": [
> {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> }
> ],
> "empty": "False",
> "type": "EC",
> "open": "False"
> },
> "replicas": [
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "ipAddress": "172.27.20.198",
> "hostName": "hostname-2.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 854432817,
> "networkLocation": "/default",
> "networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
> },
> "placeOfBirth": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 5
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "ipAddress": "172.27.87.194",
> "hostName": "hostname-4.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": 2040526203,
> "networkLocation": "/default",
> "networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "numOfLeaves": 1,
> "networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
> },
> "placeOfBirth": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 1
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
> "uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
> "ipAddress": "172.27.24.5",
> "hostName": "hostname-7.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -674050667,
> "networkLocation": "/default",
> "networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
> "numOfLeaves": 1,
> "networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
> },
> "placeOfBirth": "74078315-7c10-412b-97a2-f520ad02b91e",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 4
> },
> {
> "containerID": 8001,
> "state": "UNHEALTHY",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "ipAddress": "172.27.30.15",
> "hostName": "hostname-9.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1908661510,
> "networkLocation": "/default",
> "networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "numOfLeaves": 1,
> "networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
> },
> "placeOfBirth": "e6abd931-74f4-4a83-87ac-ad104f956706",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 54525952,
> "replicaIndex": 1
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "ipAddress": "172.27.194.199",
> "hostName": "hostname-3.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1706356826,
> "networkLocation": "/default",
> "networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "numOfLeaves": 1,
> "networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
> },
> "placeOfBirth": "18b59841-f6bd-408b-bf5d-262e58d8954b",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 53477376,
> "replicaIndex": 3
> },
> {
> "containerID": 8001,
> "state": "CLOSED",
> "datanodeDetails": {
> "level": 0,
> "cost": 0,
> "uuid": "1548294e-6375-41e1-9324-0646a8163294",
> "uuidString": "1548294e-6375-41e1-9324-0646a8163294",
> "ipAddress": "172.27.14.64",
> "hostName": "hostname-6.hostname.root.hwx.site",
> "ports": [
> {
> "name": "REPLICATION",
> "value": 9886
> },
> {
> "name": "RATIS",
> "value": 9858
> },
> {
> "name": "RATIS_ADMIN",
> "value": 9857
> },
> {
> "name": "RATIS_SERVER",
> "value": 9856
> },
> {
> "name": "STANDALONE",
> "value": 9859
> }
> ],
> "setupTime": 0,
> "persistedOpState": "IN_SERVICE",
> "persistedOpStateExpiryEpochSec": 0,
> "initialVersion": 0,
> "currentVersion": 1,
> "decomissioned": "False",
> "signature": -1098926846,
> "networkLocation": "/default",
> "networkName": "1548294e-6375-41e1-9324-0646a8163294",
> "numOfLeaves": 1,
> "networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
> },
> "placeOfBirth": "1548294e-6375-41e1-9324-0646a8163294",
> "sequenceId": 0,
> "keyCount": 1,
> "bytesUsed": 53477376,
> "replicaIndex": 2
> }
> ],
> "Datanodes": [
> "hostname-2.hostname.root.hwx.site",
> "hostname-4.hostname.root.hwx.site",
> "hostname-7.hostname.root.hwx.site",
> "hostname-9.hostname.root.hwx.site",
> "hostname-3.hostname.root.hwx.site",
> "hostname-6.hostname.root.hwx.site"
> ]
> } {code}
> Error Message from TC:
> {code:java}
> replace_status = Ozone.check_replica_present_in_container(dn_i,
> container_list[0], False)
> > assert replace_status, "Unhealthy replica not replace by healthy replica"E
> > AssertionError: Unhealthy replica not replace by healthy replica
> E assert False{code}
> DN didn't realised that the container is not there any more (eg the full
> volume has failed)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
