This is an automated email from the ASF dual-hosted git repository.
ssulav pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d0d09aacf01 HDDS-13251. Support dynamic Byteman scripts via bmsubmit
in ozonesecure-ha (#8654)
d0d09aacf01 is described below
commit d0d09aacf01e0c546ba9111fe9793aa347675eec
Author: Soumitra Sulav <[email protected]>
AuthorDate: Wed Jun 25 23:33:12 2025 +0530
HDDS-13251. Support dynamic Byteman scripts via bmsubmit in ozonesecure-ha
(#8654)
Co-authored-by: Doroszlai, Attila
<[email protected]>
---
dev-support/byteman/skip-notify-group-remove.btm | 26 +++
dev-support/byteman/skip-put-block.btm | 26 +++
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/ozonesecure-ha/.env | 2 +
.../compose/ozonesecure-ha/{.env => byteman.yaml} | 44 +++--
.../ozonesecure-ha/{.env => test-byteman.sh} | 33 ++--
.../dist/src/main/smoketest/lib/BytemanLibrary.py | 77 ++++++++
.../main/smoketest/ozone-fi/BytemanKeywords.robot | 119 ++++++++++++
.../dist/src/main/smoketest/ozone-fi/README.md | 212 +++++++++++++++++++++
.../smoketest/ozone-fi/byteman_faults_sample.robot | 60 ++++++
10 files changed, 574 insertions(+), 27 deletions(-)
diff --git a/dev-support/byteman/skip-notify-group-remove.btm
b/dev-support/byteman/skip-notify-group-remove.btm
new file mode 100644
index 00000000000..f885bab4fd1
--- /dev/null
+++ b/dev-support/byteman/skip-notify-group-remove.btm
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE skip notifyGroupRemove
+CLASS
org.apache.hadoop.ozone.container.common.transport.server.ratis.ContainerStateMachine
+METHOD notifyGroupRemove
+AT ENTRY
+IF TRUE
+DO
+ System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+ "Skip notifyGroupRemove in ContainerStateMachine");
+ return;
+ENDRULE
\ No newline at end of file
diff --git a/dev-support/byteman/skip-put-block.btm
b/dev-support/byteman/skip-put-block.btm
new file mode 100644
index 00000000000..65c46f2c6c2
--- /dev/null
+++ b/dev-support/byteman/skip-put-block.btm
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+RULE Block putBlock
+CLASS org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl
+METHOD putBlock
+AT ENTRY
+IF TRUE
+DO
+ System.out.println("[" + java.time.LocalDateTime.now() + "] BYTEMAN: " +
+ "Blocking putBlock in BlockManagerImpl");
+ return 0;
+ENDRULE
\ No newline at end of file
diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml
index 87235af34d8..b6a62358b6a 100644
--- a/hadoop-ozone/dist/pom.xml
+++ b/hadoop-ozone/dist/pom.xml
@@ -25,7 +25,7 @@
<name>Apache Ozone Distribution</name>
<properties>
<!-- suffix appended to Ozone version to get Docker image version -->
- <docker.ozone-runner.version>20241216-1-jdk21</docker.ozone-runner.version>
+ <docker.ozone-runner.version>20250625-1-jdk21</docker.ozone-runner.version>
<docker.ozone-testkr5b.image>ghcr.io/apache/ozone-testkrb5:20241129-1</docker.ozone-testkr5b.image>
<docker.ozone.image>apache/ozone</docker.ozone.image>
<docker.ozone.image.flavor>-rocky</docker.ozone.image.flavor>
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
index 75619126ca4..18011f88d37 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
@@ -27,3 +27,5 @@ RANGER_DB_IMAGE_VERSION=12
RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
RANGER_VERSION=${ranger.version}
+BYTEMAN_PORT=9091
+BYTEMAN_HOME=/opt/byteman/
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/byteman.yaml
similarity index 55%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
copy to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/byteman.yaml
index 75619126ca4..73388b9ac44 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/byteman.yaml
@@ -13,17 +13,35 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+x-byteman-config:
+ &byteman-config
+ environment:
+ BYTEMAN_HOME: /opt/byteman/
+ OZONE_SERVER_OPTS:
-javaagent:/opt/byteman.jar=listener:true,address:0.0.0.0,port:${BYTEMAN_PORT}
+ BYTEMAN_PORT: ${BYTEMAN_PORT}
-HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
-OZONE_VOLUME=./data
-OZONE_OPTS=
-RANGER_DB_IMAGE=postgres
-RANGER_DB_IMAGE_VERSION=12
-RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
-RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
-RANGER_VERSION=${ranger.version}
+services:
+ datanode1:
+ <<: *byteman-config
+ datanode2:
+ <<: *byteman-config
+ datanode3:
+ <<: *byteman-config
+ om1:
+ <<: *byteman-config
+ om2:
+ <<: *byteman-config
+ om3:
+ <<: *byteman-config
+ httpfs:
+ <<: *byteman-config
+ s3g:
+ <<: *byteman-config
+ scm1.org:
+ <<: *byteman-config
+ scm2.org:
+ <<: *byteman-config
+ scm3.org:
+ <<: *byteman-config
+ recon:
+ <<: *byteman-config
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-byteman.sh
similarity index 62%
copy from hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
copy to hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-byteman.sh
index 75619126ca4..cd1aad851c7 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/.env
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-ha/test-byteman.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,16 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-HDDS_VERSION=${hdds.version}
-HADOOP_IMAGE=apache/hadoop
-HADOOP_VERSION=${hadoop.version}
-OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
-OZONE_RUNNER_IMAGE=apache/ozone-runner
-OZONE_TESTKRB5_IMAGE=${docker.ozone-testkr5b.image}
-OZONE_VOLUME=./data
-OZONE_OPTS=
-RANGER_DB_IMAGE=postgres
-RANGER_DB_IMAGE_VERSION=12
-RANGER_IMAGE=ghcr.io/adoroszlai/ranger-admin
-RANGER_IMAGE_VERSION=0ae34250d3af672776fca6a53047699adf3afce5-${ranger.version}-8
-RANGER_VERSION=${ranger.version}
+#suite:HA-secure
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SECURITY_ENABLED=true
+export OM_SERVICE_ID="omservice"
+export SCM=scm1.org
+export COMPOSE_FILE=docker-compose.yaml:byteman.yaml
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+start_docker_env
+
+## Run virtual host test cases
+execute_robot_test om1 ozone-fi/byteman_faults_sample.robot
diff --git a/hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py
b/hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py
new file mode 100644
index 00000000000..31cf508c874
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import subprocess
+import os
+from robot.api import logger
+
+class BytemanLibrary:
+ def __init__(self):
+ self.byteman_port = os.getenv("BYTEMAN_PORT", "9091")
+ self.byteman_cmd = ["bmsubmit", "-p", self.byteman_port]
+
+ def update_component_name(self, component_name):
+ return f"{component_name}.org" if component_name in ["scm1", "scm2",
"scm3"] else component_name
+
+ def run_byteman_cmd(self, component_name, args, action_desc):
+ """Run a byteman command and handle error/logging"""
+ cmd = self.byteman_cmd + ["-h", component_name] + args
+ result = subprocess.run(cmd, capture_output=True, text=True)
+
+ if result.returncode != 0:
+ raise RuntimeError(f"Failed to {action_desc} for {component_name}:
{result.stderr.strip()}")
+
+ logger.info(f"{action_desc} for {component_name} successful.")
+ return result.stdout
+
+ def add_byteman_rule(self, component_name, rule_file):
+ """Add Byteman rule into specific component"""
+ component_name = self.update_component_name(component_name)
+ self.run_byteman_cmd(component_name, ["-l", rule_file], f"Add rule
{rule_file}")
+
+ def remove_byteman_rule(self, component_name, rule_file):
+ """Remove Byteman rule for specific component"""
+ component_name = self.update_component_name(component_name)
+ self.run_byteman_cmd(component_name, ["-u", rule_file], f"Remove rule
{rule_file}")
+
+ def list_byteman_rules(self, component_name):
+ """List Active Byteman rules for specific component and return file
list"""
+ component_name = self.update_component_name(component_name)
+ output = self.run_byteman_cmd(component_name, ["-l"], "List rules")
+
+ matching_lines = [line for line in output.splitlines() if '# File' in
line]
+ file_list = [line.split()[2] for line in matching_lines if
len(line.split()) >= 3]
+
+ if matching_lines:
+ logger.info(f"Active rules in {component_name}:\n" +
"\n".join(matching_lines))
+ else:
+ logger.info(f"Active rules in {component_name}: No rules found")
+
+ return file_list
+
+ def remove_all_byteman_rules(self, component_name):
+ """Remove all Byteman rules for specific component"""
+ component_name = self.update_component_name(component_name)
+ rule_files = self.list_byteman_rules(component_name)
+
+ if not rule_files:
+ logger.info(f"No active rules to remove for {component_name}")
+ return
+
+ for rule_file in rule_files:
+ self.remove_byteman_rule(component_name, rule_file)
diff --git
a/hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot
new file mode 100644
index 00000000000..e6ed9b6318d
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot
@@ -0,0 +1,119 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+*** Settings ***
+Library ../lib/BytemanLibrary.py
+Library String
+
+
+*** Variables ***
+@{ALL_COMPONENTS} datanode1 datanode2 datanode3 om1 om2
om3 recon scm1 scm2 scm3 s3g
+@{DATANODE_COMPONENTS} datanode1 datanode2 datanode3
+@{OM_COMPONENTS} om1 om2 om3
+@{SCM_COMPONENTS} scm1 scm2 scm3
+
+*** Keywords ***
+Inject Fault Into All Components
+ [Arguments] ${rule_file}
+ Log Injecting fault ${rule_file} into all components
+ FOR ${component} IN @{ALL_COMPONENTS}
+ Run Keyword And Continue On Failure Add Byteman Rule
${component} ${rule_file}
+ END
+
+
+Remove Fault From All Components
+ [Arguments] ${rule_file}
+ Log Removing fault ${rule_file} from all components
+ FOR ${component} IN @{ALL_COMPONENTS}
+ Run Keyword And Continue On Failure Remove Byteman Rule
${component} ${rule_file}
+ END
+
+
+List Byteman Rules for All Components
+ Log Listing active rules for all components
+ FOR ${component} IN @{ALL_COMPONENTS}
+ Run Keyword And Continue On Failure List Byteman Rules
${component}
+ END
+
+
+Remove All Rules From All Components
+ Log Removing all rules from all components
+ FOR ${component} IN @{ALL_COMPONENTS}
+ Run Keyword And Continue On Failure Remove All Byteman Rules
${component}
+ END
+
+
+Inject Fault Into Datanodes Only
+ [Arguments] ${rule_file}
+ Log Injecting fault ${rule_file} into datanodes only
+ FOR ${component} IN @{DATANODE_COMPONENTS}
+ Run Keyword And Continue On Failure Add Byteman Rule
${component} ${rule_file}
+ END
+
+List Byteman Rules for Datanodes
+ Log Listing active rules for all datanodes
+ FOR ${component} IN @{DATANODE_COMPONENTS}
+ Run Keyword And Continue On Failure List Byteman Rules
${component}
+ END
+
+Remove Fault From Datanodes Only
+ [Arguments] ${rule_file}
+ Log Removing fault ${rule_file} from datanodes only
+ FOR ${component} IN @{DATANODE_COMPONENTS}
+ Run Keyword And Continue On Failure Remove Byteman Rule
${component} ${rule_file}
+ END
+
+
+Inject Fault Into OMs Only
+ [Arguments] ${rule_file}
+ Log Injecting fault ${rule_file} into oms only
+ FOR ${component} IN @{OM_COMPONENTS}
+ Run Keyword And Continue On Failure Add Byteman Rule
${component} ${rule_file}
+ END
+
+List Byteman Rules for OMs
+ Log Listing active rules for all OMs
+ FOR ${component} IN @{OM_COMPONENTS}
+ Run Keyword And Continue On Failure List Byteman Rules
${component}
+ END
+
+Remove Fault From OMs Only
+ [Arguments] ${rule_file}
+ Log Removing fault ${rule_file} from oms only
+ FOR ${component} IN @{OM_COMPONENTS}
+ Run Keyword And Continue On Failure Remove Byteman Rule
${component} ${rule_file}
+ END
+
+
+Inject Fault Into SCMs Only
+ [Arguments] ${rule_file}
+ Log Injecting fault ${rule_file} into scms only
+ FOR ${component} IN @{SCM_COMPONENTS}
+ Run Keyword And Continue On Failure Add Byteman Rule
${component} ${rule_file}
+ END
+
+List Byteman Rules for SCMs
+ Log Listing active rules for all SCMs
+ FOR ${component} IN @{SCM_COMPONENTS}
+ Run Keyword And Continue On Failure List Byteman Rules
${component}
+ END
+
+Remove Fault From SCMs Only
+ [Arguments] ${rule_file}
+ Log Removing fault ${rule_file} from scms only
+ FOR ${component} IN @{SCM_COMPONENTS}
+ Run Keyword And Continue On Failure Remove Byteman Rule
${component} ${rule_file}
+ END
diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-fi/README.md
b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/README.md
new file mode 100644
index 00000000000..41fbf98a7cb
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/README.md
@@ -0,0 +1,212 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ozone Byteman Fault Injection Testing with Robot Framework
+
+This directory contains Robot Framework test suites for performing fault
injection testing in Apache Ozone using Byteman.
+
+## Overview
+
+Byteman is a Java bytecode manipulation tool that allows you to inject faults,
delays, and other behaviors into running Java applications without modifying
the source code. This testing framework uses Robot Framework to orchestrate
Byteman operations across Ozone cluster components.
+
+## Prerequisites
+
+- Docker and Docker Compose
+- Apache Ozone cluster running with Byteman agents enabled
+- Robot Framework (automatically installed in the test containers)
+- Byteman tools (bmsubmit) available in the test environment
+
+## Architecture
+
+### Components
+
+The fault injection framework consists of:
+
+1. **BytemanLibrary.py** - Python library providing Byteman operations
+2. **BytemanKeywords.robot** - Robot Framework keywords for common operations
+3. **Test files** - Specific test scenarios using the keywords
+
+### Supported Components
+
+- **Datanodes**: `datanode1`, `datanode2`, `datanode3`
+- **OzoneManagers**: `om1`, `om2`, `om3`
+- **StorageContainerManagers**: `scm1`, `scm2`, `scm3`
+- **Other services**: `recon`, `s3g`, `httpfs`
+
+## Usage
+
+### Basic Test Structure
+
+```robot
+*** Settings ***
+Resource BytemanKeywords.robot
+Suite Setup Setup Test Environment
+Suite Teardown Cleanup Test Environment
+
+*** Variables ***
+${RULE_FILE} /opt/hadoop/share/ozone/byteman/my-rule.btm
+
+*** Test Cases ***
+My Fault Injection Test
+ Add Byteman Rule datanode1 ${RULE_FILE}
+ # Run your test operations here
+ Remove Byteman Rule datanode1 ${RULE_FILE}
+```
+
+### Group Operations
+| Keyword | Description | Example |
+|---------|-------------|---------|
+| `Inject Fault Into All Components` | Inject rule into all components |
`Inject Fault Into All Components ${rule}` |
+| `Remove Fault From All Components` | Remove rule from all components |
`Remove Fault From All Components ${rule}` |
+| `Inject Fault Into Datanodes Only` | Inject rule into datanodes only |
`Inject Fault Into Datanodes Only ${rule}` |
+| `Inject Fault Into OMs Only` | Inject rule into OMs only | `Inject Fault
Into OMs Only ${rule}` |
+| `Inject Fault Into SCMs Only` | Inject rule into SCMs only | `Inject Fault
Into SCMs Only ${rule}` |
+
+### Individual Operations
+| Keyword | Description | Example |
+|---------|-------------|---------|
+| `Add Byteman Rule` | Add rule to specific component | `Add Byteman Rule
datanode1 ${rule}` |
+| `Remove Byteman Rule` | Remove rule from specific component | `Remove
Byteman Rule datanode1 ${rule}` |
+| `List Byteman Rules` | List active rules for component | `List Byteman Rules
datanode1` |
+| `Remove All Byteman Rules` | Remove all rules from component | `Remove All
Byteman Rules datanode1` |
+
+### Bulk Operations
+| Keyword | Description | Example |
+|---------|-------------|---------|
+| `List Byteman Rules for All Components` | List rules for all components |
`List Byteman Rules for All Components` |
+| `Remove All Rules From All Components` | Remove all rules from all
components | `Remove All Rules From All Components` |
+
+
+## Byteman Rules
+
+### Rule Location
+
+Byteman rules are stored in: `/opt/hadoop/share/ozone/byteman/`
+
+### Available Rules
+
+- `skip-put-block.btm` - Blocks putBlock operations in BlockManagerImpl
+- `skip-notify-group-remove.btm` - Skips notifyGroupRemove in
ContainerStateMachine
+- Custom rules can be added to this directory
+
+### Environment Variables
+
+- `BYTEMAN_PORT` - Port for Byteman agent communication (default: 9091)
+- `BYTEMAN_HOME` - Byteman installation directory
+
+### Component Variables
+
+The framework defines component groups:
+
+```robot
+@{ALL_COMPONENTS} datanode1 datanode2 datanode3 om1 om2 om3 recon scm1
scm2 scm3 s3g
+@{DATANODE_COMPONENTS} datanode1 datanode2 datanode3
+@{OM_COMPONENTS} om1 om2 om3
+@{SCM_COMPONENTS} scm1 scm2 scm3
+```
+
+## Running Tests
+
+### Local Development
+
+```bash
+# Navigate to compose directory
+cd hadoop-ozone/dist/src/main/compose/ozonesecure-ha
+
+# Start cluster with Byteman enabled
+export COMPOSE_FILE=docker-compose.yaml:byteman.yaml
+docker-compose up -d
+
+# Run fault injection tests
+./test-byteman.sh
+```
+
+### CI/CD Integration
+
+```bash
+# Run specific test suite
+execute_robot_test om1 ozone-fi/byteman_faults_sample.robot
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **bmsubmit not found**
+ - Ensure Byteman is properly installed in the container
+ - Check PATH includes `/usr/local/bin`
+
+2. **Connection refused**
+ - Verify Byteman agents are running on target components
+ - Check BYTEMAN_PORT configuration
+
+3. **Rule file not found**
+ - Ensure rule files are mounted in the container
+ - Verify path `/opt/hadoop/share/ozone/byteman/`
+
+4. **Classpath errors**
+ - Rebuild Ozone distribution: `mvn clean install -DskipTests`
+ - Ensure all required JARs are present
+
+### Debugging
+
+Enable verbose logging:
+
+```robot
+*** Settings ***
+Library BytemanLibrary.py WITH NAME Byteman
+Library OperatingSystem
+
+*** Test Cases ***
+Debug Byteman Operations
+ ${output} = Execute bmsubmit -p 9091 -h datanode1 -l
+ Log ${output}
+```
+
+### Validation
+
+Check if Byteman agents are running:
+
+```bash
+# Check if agent is listening
+docker exec datanode1 netstat -ln | grep 9091
+
+# List active rules
+docker exec datanode1 bmsubmit -p 9091 -l
+```
+
+## Best Practices
+
+1. **Always clean up** - Use Suite Teardown to remove injected faults
+2. **Use specific components** - Target specific services rather than all
components when possible
+3. **Error handling** - Use `Run Keyword And Continue On Failure` for bulk
operations
+4. **Documentation** - Document expected behavior and fault scenarios
+5. **Isolation** - Ensure tests don't interfere with each other
+
+## Contributing
+
+When adding new test cases:
+
+1. Follow the existing keyword naming conventions
+2. Add appropriate documentation strings
+3. Include both positive and negative test scenarios
+4. Ensure proper cleanup in teardown methods
+5. Test with different cluster configurations
+
+## References
+
+- [Byteman Documentation](http://byteman.jboss.org/)
+- [Robot Framework User
Guide](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html)
+- [Apache Ozone Documentation](https://ozone.apache.org/)
\ No newline at end of file
diff --git
a/hadoop-ozone/dist/src/main/smoketest/ozone-fi/byteman_faults_sample.robot
b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/byteman_faults_sample.robot
new file mode 100644
index 00000000000..0f32f39cec8
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/ozone-fi/byteman_faults_sample.robot
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+*** Variables ***
+${RULE1} /opt/hadoop/share/ozone/byteman/skip-put-block.btm
+${RULE2} /opt/hadoop/share/ozone/byteman/skip-notify-group-remove.btm
+
+*** Settings ***
+Resource BytemanKeywords.robot
+
+
+*** Test Cases ***
+
+Print All Byteman Rules
+ Inject Fault Into All Components ${RULE1}
+ List Byteman Rules for All Components
+ Remove Fault From All Components ${RULE1}
+
+Inject Byteman Rule in one component
+ Add Byteman Rule datanode1 ${RULE2}
+ List Byteman Rules datanode1
+ Remove Byteman Rule datanode1 ${RULE2}
+
+Inject Multiple Byteman Rules in one component
+ Add Byteman Rule datanode1 ${RULE1}
+ Add Byteman Rule datanode1 ${RULE2}
+ ${rules} = List Byteman Rules datanode1
+ ${rules_count} = Get Length ${rules}
+ Should Be Equal As Integers ${rules_count} 2
+ Remove All Byteman Rules datanode1
+ ${rules} = List Byteman Rules datanode1
+ Should Be Empty ${rules}
+
+Test Datanode Only Fault Injection
+ Inject Fault Into Datanodes Only ${RULE1}
+ List Byteman Rules for Datanodes
+ Remove Fault From Datanodes Only ${RULE1}
+
+Test OM Only Fault Injection
+ Inject Fault Into OMs Only ${RULE1}
+ List Byteman Rules for OMs
+ Remove Fault From OMs Only ${RULE1}
+
+Test SCM Only Fault Injection
+ Inject Fault Into SCMs Only ${RULE1}
+ List Byteman Rules for SCMs
+ Remove Fault From SCMs Only ${RULE1}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
