Arun Sarin created HDDS-8441:
--------------------------------
Summary: [EC] Unhealthy replica not getting replaced by healthy
replica when hdds.datanode.dir path is deleted
Key: HDDS-8441
URL: https://issues.apache.org/jira/browse/HDDS-8441
Project: Apache Ozone
Issue Type: Bug
Reporter: Arun Sarin
Scenario Repro :
# Close container of a key
# Simulate one unhealthy replica in above container by deleting
hdds.datanode.dir
# New replica is added for the unhealthy replica but after successful addition
the unhealthy replica is not removed
Final state of container - 2 replicas on replica index 1
{code:java}
{
"containerInfo": {
"state": "CLOSED",
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"usedBytes": 54525952,
"numberOfKeys": 1,
"lastUsed": "2023-04-11T11:10:42.412Z",
"stateEnterTime": "2023-04-11T10:54:42.254Z",
"owner": "om3",
"containerID": 8001,
"deleteTransactionId": 0,
"sequenceId": 0,
"deleted": "False",
"open": "False"
},
"pipeline": {
"id": {
"id": "10064036-6251-40a9-b4fa-e1edeae388e5"
},
"replicationConfig": {
"data": 3,
"parity": 2,
"ecChunkSize": 1048576,
"codec": "RS",
"replicationType": "EC",
"requiredNodes": 5
},
"nodesInOrder": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"creationTimestamp": "2023-04-11T11:10:42.344Z",
"closed": "False",
"allocationTimeout": "False",
"healthy": "True",
"pipelineState": "ALLOCATED",
"nodeSet": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"leaderNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"firstNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"closestNode": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"nodes": [
{
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
{
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
{
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
{
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
{
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
{
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
}
],
"empty": "False",
"type": "EC",
"open": "False"
},
"replicas": [
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"uuidString": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"ipAddress": "172.27.20.198",
"hostName": "hostname-2.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 854432817,
"networkLocation": "/default",
"networkName": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"numOfLeaves": 1,
"networkFullPath": "/default/5282b59f-e8bf-44a1-9a6b-19151d717ea2"
},
"placeOfBirth": "5282b59f-e8bf-44a1-9a6b-19151d717ea2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 5
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"uuidString": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"ipAddress": "172.27.87.194",
"hostName": "hostname-4.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": 2040526203,
"networkLocation": "/default",
"networkName": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"numOfLeaves": 1,
"networkFullPath": "/default/0781b6a5-36bc-4310-83ff-c9fc6ada70b2"
},
"placeOfBirth": "0781b6a5-36bc-4310-83ff-c9fc6ada70b2",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "74078315-7c10-412b-97a2-f520ad02b91e",
"uuidString": "74078315-7c10-412b-97a2-f520ad02b91e",
"ipAddress": "172.27.24.5",
"hostName": "hostname-7.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -674050667,
"networkLocation": "/default",
"networkName": "74078315-7c10-412b-97a2-f520ad02b91e",
"numOfLeaves": 1,
"networkFullPath": "/default/74078315-7c10-412b-97a2-f520ad02b91e"
},
"placeOfBirth": "74078315-7c10-412b-97a2-f520ad02b91e",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 4
},
{
"containerID": 8001,
"state": "UNHEALTHY",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "e6abd931-74f4-4a83-87ac-ad104f956706",
"uuidString": "e6abd931-74f4-4a83-87ac-ad104f956706",
"ipAddress": "172.27.30.15",
"hostName": "hostname-9.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1908661510,
"networkLocation": "/default",
"networkName": "e6abd931-74f4-4a83-87ac-ad104f956706",
"numOfLeaves": 1,
"networkFullPath": "/default/e6abd931-74f4-4a83-87ac-ad104f956706"
},
"placeOfBirth": "e6abd931-74f4-4a83-87ac-ad104f956706",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 54525952,
"replicaIndex": 1
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"uuidString": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"ipAddress": "172.27.194.199",
"hostName": "hostname-3.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1706356826,
"networkLocation": "/default",
"networkName": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"numOfLeaves": 1,
"networkFullPath": "/default/18b59841-f6bd-408b-bf5d-262e58d8954b"
},
"placeOfBirth": "18b59841-f6bd-408b-bf5d-262e58d8954b",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 3
},
{
"containerID": 8001,
"state": "CLOSED",
"datanodeDetails": {
"level": 0,
"cost": 0,
"uuid": "1548294e-6375-41e1-9324-0646a8163294",
"uuidString": "1548294e-6375-41e1-9324-0646a8163294",
"ipAddress": "172.27.14.64",
"hostName": "hostname-6.hostname.root.hwx.site",
"ports": [
{
"name": "REPLICATION",
"value": 9886
},
{
"name": "RATIS",
"value": 9858
},
{
"name": "RATIS_ADMIN",
"value": 9857
},
{
"name": "RATIS_SERVER",
"value": 9856
},
{
"name": "STANDALONE",
"value": 9859
}
],
"setupTime": 0,
"persistedOpState": "IN_SERVICE",
"persistedOpStateExpiryEpochSec": 0,
"initialVersion": 0,
"currentVersion": 1,
"decomissioned": "False",
"signature": -1098926846,
"networkLocation": "/default",
"networkName": "1548294e-6375-41e1-9324-0646a8163294",
"numOfLeaves": 1,
"networkFullPath": "/default/1548294e-6375-41e1-9324-0646a8163294"
},
"placeOfBirth": "1548294e-6375-41e1-9324-0646a8163294",
"sequenceId": 0,
"keyCount": 1,
"bytesUsed": 53477376,
"replicaIndex": 2
}
],
"Datanodes": [
"hostname-2.hostname.root.hwx.site",
"hostname-4.hostname.root.hwx.site",
"hostname-7.hostname.root.hwx.site",
"hostname-9.hostname.root.hwx.site",
"hostname-3.hostname.root.hwx.site",
"hostname-6.hostname.root.hwx.site"
]
} {code}
Error Message from TC:
replace_status = Ozone.check_replica_present_in_container(dn_i,
container_list[0], False)
> assert replace_status, "Unhealthy replica not replace by healthy
> replica"E AssertionError: Unhealthy replica not replace by healthy
> replica
E assert False
DN didn't realised that the container is not there any more (eg the full volume
has failed)
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
