This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch ozone-0.4.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/ozone-0.4.1 by this push:
new 7cdb80b HDDS-1853. Fix failing blockade test-cases. (#1151)
7cdb80b is described below
commit 7cdb80b16254355d0e7546b559a2862becb67218
Author: Nanda kumar <[email protected]>
AuthorDate: Thu Jul 25 00:02:28 2019 +0530
HDDS-1853. Fix failing blockade test-cases. (#1151)
(cherry picked from commit cb69700ac6b535e108b43f00a61f31712f2cecb2)
---
.../src/main/compose/ozoneblockade/docker-config | 6 ++--
.../src/test/blockade/ozone/cluster.py | 7 +++++
.../src/test/blockade/ozone/container.py | 34 +++++++++++++++++-----
.../test/blockade/test_blockade_client_failure.py | 8 +++--
.../blockade/test_blockade_datanode_isolation.py | 1 +
.../test/blockade/test_blockade_mixed_failure.py | 2 ++
...t_blockade_mixed_failure_three_nodes_isolate.py | 2 +-
.../test_blockade_mixed_failure_two_nodes.py | 2 ++
.../test/blockade/test_blockade_scm_isolation.py | 7 +++--
9 files changed, 52 insertions(+), 17 deletions(-)
diff --git a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
index 8347998..af72465 100644
--- a/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozoneblockade/docker-config
@@ -32,9 +32,9 @@ OZONE-SITE.XML_ozone.scm.pipeline.owner.container.count=1
OZONE-SITE.XML_ozone.scm.pipeline.destroy.timeout=15s
OZONE-SITE.XML_hdds.heartbeat.interval=2s
OZONE-SITE.XML_hdds.scm.wait.time.after.safemode.exit=30s
-OZONE-SITE.XML_hdds.scm.replication.thread.interval=5s
-OZONE-SITE.XML_hdds.scm.replication.event.timeout=7s
-OZONE-SITE.XML_dfs.ratis.server.failure.duration=25s
+OZONE-SITE.XML_hdds.scm.replication.thread.interval=6s
+OZONE-SITE.XML_hdds.scm.replication.event.timeout=10s
+OZONE-SITE.XML_dfs.ratis.server.failure.duration=35s
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
index d137793..9888e86 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/cluster.py
@@ -146,6 +146,11 @@ class OzoneCluster(object):
"""
Start Ozone Cluster in docker containers.
"""
+ # check if proper env $HDDS_VERSION and $HADOOP_RUNNER_VERSION
+ # are set.
+
+ # check if docker is up.
+
self.__logger__.info("Starting Ozone Cluster")
if Blockade.blockade_status() == 0:
Blockade.blockade_destroy()
@@ -263,6 +268,8 @@ class OzoneCluster(object):
# Reading the container file.
exit_code, output = util.run_docker_command("cat " + container_path,
datanode)
+ if exit_code != 0:
+ raise ContainerNotFoundError("Container not found!")
data = output.split("\n")
# Reading key value pairs from container file.
key_value = [x for x in data if re.search(r"\w+:\s\w+", x)]
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/container.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/container.py
index ffb6a3d..6e8c344 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/container.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/ozone/container.py
@@ -83,20 +83,37 @@ class Container:
for dn in dns:
if self.cluster.get_container_state(self.container_id, dn) ==
'CLOSED':
return True
- else:
- return False
+ return False
util.wait_until(predicate, int(os.environ["CONTAINER_STATUS_SLEEP"]),
10)
if not predicate():
raise Exception("None of the container replica is closed!")
- def wait_until_all_replicas_are_closed(self):
+ def wait_until_two_replicas_are_closed(self):
def predicate():
dns = self.cluster.get_container_datanodes(self.container_id)
+ closed_count = 0
for dn in dns:
- if self.cluster.get_container_state(self.container_id, dn) !=
'CLOSED':
- return False
- return True
+ if self.cluster.get_container_state(self.container_id, dn) ==
'CLOSED':
+ closed_count = closed_count + 1
+ if closed_count > 1:
+ return True
+ return False
+
+ util.wait_until(predicate, int(os.environ["CONTAINER_STATUS_SLEEP"]),
10)
+ if not predicate():
+ raise Exception("None of the container replica is closed!")
+
+ def wait_until_all_replicas_are_closed(self):
+ def predicate():
+ try:
+ dns = self.cluster.get_container_datanodes(self.container_id)
+ for dn in dns:
+ if self.cluster.get_container_state(self.container_id, dn)
!= 'CLOSED':
+ return False
+ return True
+ except ContainerNotFoundError:
+ return False
util.wait_until(predicate, int(os.environ["CONTAINER_STATUS_SLEEP"]),
10)
if not predicate():
@@ -105,7 +122,8 @@ class Container:
def wait_until_replica_is_not_open_anymore(self, datanode):
def predicate():
try:
- if self.cluster.get_container_state(self.container_id,
datanode) != 'OPEN':
+ if self.cluster.get_container_state(self.container_id,
datanode) != 'OPEN' and \
+ self.cluster.get_container_state(self.container_id,
datanode) != 'CLOSING':
return True
else:
return False
@@ -114,4 +132,4 @@ class Container:
util.wait_until(predicate, int(os.environ["CONTAINER_STATUS_SLEEP"]),
10)
if not predicate():
- raise Exception("Replica is not closed!")
\ No newline at end of file
+ raise Exception("Replica is not closed!")
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
index 55b5291..beb192f 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_client_failure.py
@@ -19,6 +19,7 @@ import re
import time
import logging
import ozone.util
+import pytest
from ozone.cluster import OzoneCluster
@@ -35,6 +36,8 @@ def teardown_function():
cluster.stop()
[email protected](reason="The test-case fails intermittently."
+ "See HDDS-1817 for more info.")
def test_client_failure_isolate_two_datanodes():
"""
In this test, all DNs are isolated from each other.
@@ -66,7 +69,7 @@ def test_client_failure_isolate_two_datanodes():
cluster.partition_network(first_set, second_set, third_set)
exit_code, output = oz_client.run_freon(1, 1, 1, 10240)
- assert re.search("Status: Failed", output) is not None
+ assert exit_code != 0, "freon run should have failed."
oz_client.get_key(volume_name, bucket_name, key_name, "/tmp/")
@@ -76,6 +79,7 @@ def test_client_failure_isolate_two_datanodes():
assert file_checksum == key_checksum
[email protected](reason="HDDS-1817")
def test_client_failure_isolate_one_datanode():
"""
In this test, one of the DNs is isolated from all other nodes.
@@ -106,7 +110,7 @@ def test_client_failure_isolate_one_datanode():
exit_code, output = oz_client.run_freon(1, 1, 1, 10240)
assert re.search("3 way commit failed", output) is not None
- assert re.search("Status: Success", output) is not None
+ assert exit_code == 0, "freon run failed with output=[%s]" % output
oz_client.get_key(volume_name, bucket_name, key_name, "/tmp/")
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
index 5c19116..66d6579 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_datanode_isolation.py
@@ -33,6 +33,7 @@ def teardown_function():
cluster.stop()
[email protected](reason="HDDS-1850")
def test_isolate_single_datanode():
"""
In this test case we will create a network partition in such a way that
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
index 10220b9..6e943c1 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure.py
@@ -16,6 +16,7 @@
# limitations under the License.
import logging
+import pytest
from ozone.cluster import OzoneCluster
@@ -72,6 +73,7 @@ def test_one_dn_isolate_scm_other_dn():
assert exit_code == 0, "freon run failed with output=[%s]" % output
[email protected](reason="HDDS-1850")
def test_one_dn_isolate_other_dn():
"""
In this test, one of the DNs (first DN) cannot communicate
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
index d213a22..6f01c84 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_three_nodes_isolate.py
@@ -100,7 +100,7 @@ def test_three_dns_isolate_two_scm_failure():
cluster.partition_network(first_set, second_set, third_set)
containers = cluster.get_containers_on_datanode(dns[0])
for container in containers:
- container.wait_until_replica_is_closed(dns[0])
+ container.wait_until_replica_is_quasi_closed(dns[0])
for container in containers:
assert container.get_state(dns[0]) == 'QUASI_CLOSED'
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
index 20b0cc3..a17947d 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_mixed_failure_two_nodes.py
@@ -16,6 +16,7 @@
# limitations under the License.
import logging
+import pytest
from ozone.cluster import OzoneCluster
@@ -32,6 +33,7 @@ def teardown_function():
cluster.stop()
[email protected](reason="HDDS-1850")
def test_two_dns_isolate_scm_same_partition():
"""
In this test, there are three DNs,
diff --git
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
index b6ca5a4..f48ddf3 100644
---
a/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
+++
b/hadoop-ozone/fault-injection-test/network-tests/src/test/blockade/test_blockade_scm_isolation.py
@@ -55,12 +55,13 @@ def test_scm_isolation_one_node():
containers = cluster.get_containers_on_datanode(dns[1])
for container in containers:
- container.wait_until_one_replica_is_closed()
+ container.wait_until_two_replicas_are_closed()
for container in containers:
- assert container.get_state(dns[0]) == 'OPEN'
assert container.get_state(dns[1]) == 'CLOSED'
assert container.get_state(dns[2]) == 'CLOSED'
+ assert container.get_state(dns[0]) == 'OPEN' or \
+ container.get_state(dns[0]) == 'CLOSED'
cluster.restore_network()
@@ -107,7 +108,7 @@ def test_scm_isolation_two_node():
if state == 'QUASI_CLOSED':
assert container.get_state(dns[0]) == 'OPEN'
assert container.get_state(dns[2]) == 'OPEN'
- else :
+ else:
assert container.get_state(dns[0]) == 'CLOSED'
assert container.get_state(dns[2]) == 'CLOSED'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
