This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3328d7d HDDS-2720. Ozone Failure injection Service (#956)
3328d7d is described below
commit 3328d7dc706b3a098a10def34aa367f4a7414b3e
Author: prashantpogde <[email protected]>
AuthorDate: Mon Jun 8 11:26:37 2020 -0700
HDDS-2720. Ozone Failure injection Service (#956)
---
tools/fault-injection-service/CMakeLists.txt | 114 +++
.../FileSystem/cpp_unit/TestFailureInjector.cc | 175 ++++
.../FileSystem/cpp_unit/TestFilePathFailures.cc | 138 ++++
.../FileSystem/failure_injector.cc | 201 +++++
.../FileSystem/failure_injector.h | 145 ++++
.../FileSystem/failure_injector_fs.cc | 912 +++++++++++++++++++++
.../FileSystem/failure_injector_fs.h | 176 ++++
tools/fault-injection-service/README.md | 56 ++
.../Service/cpp/failure_injector_svc_client.cc | 209 +++++
.../Service/cpp/failure_injector_svc_client.h | 51 ++
.../Service/cpp/failure_injector_svc_server.cc | 93 +++
.../Service/cpp/failure_injector_svc_server.h | 63 ++
.../Service/cpp/run_grpc_service.cc | 58 ++
.../Service/cpp/run_grpc_service.h | 36 +
.../Service/protos/failure_injection_service.proto | 68 ++
15 files changed, 2495 insertions(+)
diff --git a/tools/fault-injection-service/CMakeLists.txt
b/tools/fault-injection-service/CMakeLists.txt
new file mode 100644
index 0000000..ebb3cad
--- /dev/null
+++ b/tools/fault-injection-service/CMakeLists.txt
@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with this
+# work for additional information regarding copyright ownership. The ASF
+# licenses this file to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# <p>
+# http://www.apache.org/licenses/LICENSE-2.0
+# <p>
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+# cmake build file for C++ failure_injection_service.
+# Assumes protobuf and gRPC have been installed using cmake.
+
+cmake_minimum_required(VERSION 2.8)
+
+project(FailureInjectionService C CXX)
+
+set(BASE_DIR ".")
+set(FS_DIR "${BASE_DIR}/FileSystem")
+set(SRV_DIR "${BASE_DIR}/Service/cpp")
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fpermissive -Wall")
+
+# This assumes that gRPC and all its dependencies are already installed
+# on this system, so they can be located by find_package().
+
+# Find Protobuf installation
+# Looks for protobuf-config.cmake file installed by Protobuf's cmake
+# installation.
+set(protobuf_MODULE_COMPATIBLE TRUE)
+find_package(Protobuf CONFIG REQUIRED)
+message(STATUS "Using protobuf ${protobuf_VERSION}")
+
+set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+
+# Find gRPC installation
+# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+find_package(gRPC CONFIG REQUIRED)
+message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+set(_GRPC_GRPCPP_UNSECURE gRPC::grpc++_unsecure)
+set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+
+# Proto file
+get_filename_component(hw_proto
+ "${BASE_DIR}/Service/protos/failure_injection_service.proto"
+ ABSOLUTE)
+get_filename_component(hw_proto_path "${hw_proto}" PATH)
+
+# Generated sources
+set(hw_proto_srcs
+ "${CMAKE_CURRENT_BINARY_DIR}/failure_injection_service.pb.cc")
+set(hw_proto_hdrs
+ "${CMAKE_CURRENT_BINARY_DIR}/failure_injection_service.pb.h")
+set(hw_grpc_srcs
+ "${CMAKE_CURRENT_BINARY_DIR}/failure_injection_service.grpc.pb.cc")
+set(hw_grpc_hdrs
+ "${CMAKE_CURRENT_BINARY_DIR}/failure_injection_service.grpc.pb.h")
+add_custom_command(
+ OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}"
+ "${hw_grpc_srcs}" "${hw_grpc_hdrs}"
+ COMMAND ${_PROTOBUF_PROTOC}
+ ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+ --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+ -I "${hw_proto_path}"
+ --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+ "${hw_proto}"
+ DEPENDS "${hw_proto}")
+
+# Include generated *.pb.h files
+include_directories("${CMAKE_CURRENT_BINARY_DIR}"
+ "${FS_DIR}"
+ "${SRV_DIR}")
+
+#add_compile_options("-fpermissive")
+
+# Build server
+add_executable(failure_injector_svc_server
+ ${FS_DIR}/failure_injector_fs.cc ${FS_DIR}/failure_injector.cc
+ ${SRV_DIR}/failure_injector_svc_server.cc ${SRV_DIR}/run_grpc_service.cc
+ ${hw_proto_srcs}
+ ${hw_grpc_srcs})
+target_link_libraries(failure_injector_svc_server
+ ${_GRPC_GRPCPP_UNSECURE}
+ fuse3
+ ${_PROTOBUF_LIBPROTOBUF})
+
+# Build client
+add_executable(failure_injector_svc_client
+ "${SRV_DIR}/failure_injector_svc_client.cc"
+ ${hw_proto_srcs}
+ ${hw_grpc_srcs})
+target_link_libraries(failure_injector_svc_client
+ ${_GRPC_GRPCPP_UNSECURE}
+ ${_PROTOBUF_LIBPROTOBUF})
+
+# Build unit tests
+set(CPP_UNIT_FUSE cpp_unit)
+foreach(_target
+ TestFilePathFailures
+ TestFailureInjector)
+ add_executable(${_target}
+ "${FS_DIR}/${CPP_UNIT_FUSE}/${_target}.cc"
+ ${FS_DIR}/failure_injector.cc)
+ target_link_libraries(${_target}
+ cppunit)
+endforeach()
diff --git
a/tools/fault-injection-service/FileSystem/cpp_unit/TestFailureInjector.cc
b/tools/fault-injection-service/FileSystem/cpp_unit/TestFailureInjector.cc
new file mode 100644
index 0000000..c501d8e
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/cpp_unit/TestFailureInjector.cc
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include <iostream>
+#include <string>
+#include <list>
+#include <cppunit/TestCase.h>
+#include <cppunit/TestFixture.h>
+#include <cppunit/ui/text/TextTestRunner.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/TestResult.h>
+#include <cppunit/TestResultCollector.h>
+#include <cppunit/TestRunner.h>
+#include <cppunit/BriefTestProgressListener.h>
+#include <cppunit/CompilerOutputter.h>
+#include <cppunit/XmlOutputter.h>
+#include <netinet/in.h>
+#include "failure_injector.h"
+
+using namespace CppUnit;
+using namespace std;
+using namespace NoiseInjector;
+
+//-----------------------------------------------------------------------------
+
+class TestFailureInjector : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestFailureInjector);
+ CPPUNIT_TEST(testFailureInjector);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+ void setUp(void);
+ void tearDown(void);
+
+protected:
+ void testFailureInjector();
+ void testGetOpCode();
+
+private:
+
+ FailureInjector *mTestObj;
+};
+
+//-----------------------------------------------------------------------------
+
+void TestFailureInjector::testGetOpCode()
+{
+ CPPUNIT_ASSERT(mTestObj->getOpCode("NO_OP") == NO_OP);
+ CPPUNIT_ASSERT(mTestObj->getOpCode("GETATTR") == GETATTR);
+ CPPUNIT_ASSERT(mTestObj->getOpCode("READDIR") == READDIR);
+ CPPUNIT_ASSERT(mTestObj->getOpCode("LSEEK") == LSEEK);
+ CPPUNIT_ASSERT(mTestObj->getOpCode("NUM_OP_CODES") == NUM_OP_CODES);
+}
+
+//-----------------------------------------------------------------------------
+
+void TestFailureInjector::testFailureInjector()
+{
+ string path("/opt/test.xyz");
+ string path2("/opt/test2.xyz");
+
+ testGetOpCode();
+ mTestObj->InjectFailure(path, "RENAME",
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "RENAME")->size() == 1);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(path, "RENAME"))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+
+ // two actions for (path, RENAME)
+ mTestObj->InjectFailure(path, "RENAME",
+ InjectedAction(InjectedAction::ActionCode::FAIL,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "RENAME")->size() == 2);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(path, "RENAME"))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(path, "RENAME"))[1].code ==
+ InjectedAction::ActionCode::FAIL);
+
+ // one actions for (path,READ)
+ mTestObj->InjectFailure(path, "READ",
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "READ")->size() == 1);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(path, "READ"))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+
+ // one actions for (path2,LOCK)
+ mTestObj->InjectFailure(path2, "LOCK",
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path2, "LOCK")->size() == 1);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(path2, "LOCK"))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+
+ mTestObj->dumpFailures();
+
+ mTestObj->ResetFailure(path, READ);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "READ") == NULL);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "RENAME")->size() == 2);
+
+ mTestObj->ResetFailure(path);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "RENAME") == NULL);
+
+ mTestObj->ResetFailure();
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "RENAME") == NULL);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path, "READ") == NULL);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(path2, "LOCK") == NULL);
+
+ mTestObj->dumpFailures();
+}
+
+//-----------------------------------------------------------------------------
+
+void TestFailureInjector::setUp(void)
+{
+ mTestObj = new FailureInjector();
+}
+
+void TestFailureInjector::tearDown(void)
+{
+ delete mTestObj;
+}
+
+//-----------------------------------------------------------------------------
+
+CPPUNIT_TEST_SUITE_REGISTRATION( TestFailureInjector );
+
+int main(int argc, char* argv[])
+{
+ // informs test-listener about testresults
+ CPPUNIT_NS::TestResult testresult;
+
+ // register listener for collecting the test-results
+ CPPUNIT_NS::TestResultCollector collectedresults;
+ testresult.addListener (&collectedresults);
+
+ // register listener for per-test progress output
+ CPPUNIT_NS::BriefTestProgressListener progress;
+ testresult.addListener (&progress);
+
+ // insert test-suite at test-runner by registry
+ CPPUNIT_NS::TestRunner testrunner;
+ testrunner.addTest
(CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest ());
+ testrunner.run(testresult);
+
+ // output results in compiler-format
+ CPPUNIT_NS::CompilerOutputter compileroutputter(&collectedresults,
std::cerr);
+ compileroutputter.write ();
+
+ // Output XML for Jenkins CPPunit plugin
+ ofstream xmlFileOut("cppTestFailureInjectorResults.xml");
+ XmlOutputter xmlOut(&collectedresults, xmlFileOut);
+ xmlOut.write();
+
+ // return 0 if tests were successful
+ return collectedresults.wasSuccessful() ? 0 : 1;
+}
+
diff --git
a/tools/fault-injection-service/FileSystem/cpp_unit/TestFilePathFailures.cc
b/tools/fault-injection-service/FileSystem/cpp_unit/TestFilePathFailures.cc
new file mode 100644
index 0000000..9b681bc
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/cpp_unit/TestFilePathFailures.cc
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include <iostream>
+#include <string>
+#include <list>
+#include <cppunit/TestCase.h>
+#include <cppunit/TestFixture.h>
+#include <cppunit/ui/text/TextTestRunner.h>
+#include <cppunit/extensions/HelperMacros.h>
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/TestResult.h>
+#include <cppunit/TestResultCollector.h>
+#include <cppunit/TestRunner.h>
+#include <cppunit/BriefTestProgressListener.h>
+#include <cppunit/CompilerOutputter.h>
+#include <cppunit/XmlOutputter.h>
+#include <netinet/in.h>
+#include "failure_injector.h"
+
+using namespace CppUnit;
+using namespace std;
+using namespace NoiseInjector;
+
+//-----------------------------------------------------------------------------
+
+class TestFilePathFailures : public CppUnit::TestFixture
+{
+ CPPUNIT_TEST_SUITE(TestFilePathFailures);
+ CPPUNIT_TEST(testFilePathFailures);
+ CPPUNIT_TEST_SUITE_END();
+
+public:
+ void setUp(void);
+ void tearDown(void);
+
+protected:
+ void testFilePathFailures();
+
+private:
+
+ FilePathFailures *mTestObj;
+};
+
+//-----------------------------------------------------------------------------
+
+void TestFilePathFailures::testFilePathFailures()
+{
+ mTestObj->addFailure(READ,
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(READ)->size() == 1);
+
+ mTestObj->addFailure(READ,
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ // Redundant actions should not be added
+ CPPUNIT_ASSERT(mTestObj->GetFailures(READ)->size() == 1);
+ CPPUNIT_ASSERT(mTestObj->GetFailures(READ)->size() == 1);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(READ))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+ mTestObj->addFailure(READ,
+ InjectedAction(InjectedAction::ActionCode::FAIL,
+ 1, 0));
+ mTestObj->addFailure(READ,
+ InjectedAction(InjectedAction::ActionCode::CORRUPT,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(READ)->size() == 3);
+ mTestObj->addFailure(FLOCK,
+ InjectedAction(InjectedAction::ActionCode::DELAY,
+ 1, 0));
+ CPPUNIT_ASSERT(mTestObj->GetFailures(FLOCK)->size() == 1);
+ CPPUNIT_ASSERT((*mTestObj->GetFailures(FLOCK))[0].code ==
+ InjectedAction::ActionCode::DELAY);
+ mTestObj->dumpFailures();
+}
+
+//-----------------------------------------------------------------------------
+
+void TestFilePathFailures::setUp(void)
+{
+ mTestObj = new FilePathFailures();
+}
+
+void TestFilePathFailures::tearDown(void)
+{
+ delete mTestObj;
+}
+
+//-----------------------------------------------------------------------------
+
+CPPUNIT_TEST_SUITE_REGISTRATION( TestFilePathFailures );
+
+int main(int argc, char* argv[])
+{
+ // informs test-listener about testresults
+ CPPUNIT_NS::TestResult testresult;
+
+ // register listener for collecting the test-results
+ CPPUNIT_NS::TestResultCollector collectedresults;
+ testresult.addListener (&collectedresults);
+
+ // register listener for per-test progress output
+ CPPUNIT_NS::BriefTestProgressListener progress;
+ testresult.addListener (&progress);
+
+ // insert test-suite at test-runner by registry
+ CPPUNIT_NS::TestRunner testrunner;
+ testrunner.addTest
(CPPUNIT_NS::TestFactoryRegistry::getRegistry().makeTest ());
+ testrunner.run(testresult);
+
+ // output results in compiler-format
+ CPPUNIT_NS::CompilerOutputter compileroutputter(&collectedresults,
std::cerr);
+ compileroutputter.write ();
+
+ // Output XML for Jenkins CPPunit plugin
+ ofstream xmlFileOut("cppTestFilePathFailuresResults.xml");
+ XmlOutputter xmlOut(&collectedresults, xmlFileOut);
+ xmlOut.write();
+
+ // return 0 if tests were successful
+ return collectedresults.wasSuccessful() ? 0 : 1;
+}
+
diff --git a/tools/fault-injection-service/FileSystem/failure_injector.cc
b/tools/fault-injection-service/FileSystem/failure_injector.cc
new file mode 100644
index 0000000..2c6928a
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/failure_injector.cc
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include<string>
+#include<map>
+#include<vector>
+#include<iostream>
+#include"failure_injector.h"
+
+using namespace std;
+using namespace NoiseInjector;
+
+#define GEN_CODE_NAMES(x) #x
+
+string InjectedOpNames[] = {"NO_OP", InjectedOperations(GEN_CODE_NAMES)};
+
+vector<InjectedAction> *FilePathFailures::GetFailures(InjectedOpCode op)
+{
+ auto iter = mInjectedOperations.find(op);
+ if (iter == mInjectedOperations.end()) {
+ return NULL;
+ }
+ return (vector<InjectedAction> *)&iter->second;
+}
+
+void FilePathFailures::addFailure(
+ InjectedOpCode op,
+ InjectedAction action)
+{
+ auto iter = mInjectedOperations.find(op);
+ if (iter == mInjectedOperations.end()) {
+ mInjectedOperations[op].push_back(action);
+ } else {
+ vector<InjectedAction> &action_vector = iter->second;
+ for (auto &existing_action : action_vector) {
+ if (action.code == existing_action.code) {
+ return;
+ }
+ }
+ action_vector.push_back(action);
+ }
+}
+
+void FilePathFailures::resetFailure(InjectedOpCode op)
+{
+ auto itr = mInjectedOperations.find(op);
+ if (itr != mInjectedOperations.end()) {
+ mInjectedOperations[op].clear();
+ mInjectedOperations.erase(op);
+ }
+}
+
+void FilePathFailures::resetFailure()
+{
+ for (auto iter:mInjectedOperations) {
+ InjectedOpCode op = iter.first;
+ mInjectedOperations[op].clear();
+ mInjectedOperations.erase(op);
+ }
+}
+
+void FilePathFailures::dumpFailures()
+{
+ for (auto iter:mInjectedOperations) {
+ InjectedOpCode op = iter.first;
+ for (const auto action : mInjectedOperations[op]) {
+ cout << "\t" << InjectedOpNames[op] << ": " << action.code << "\n";
+ }
+ }
+}
+
+InjectedOpCode FailureInjector::getOpCode(std::string op_name)
+{
+ int i = 0;
+ for (auto name : InjectedOpNames) {
+ if (!name.compare(op_name)) {
+ return (InjectedOpCode)i;
+ }
+ ++i;
+ }
+ return NO_OP;
+}
+
+std::vector<InjectedAction>* FailureInjector::GetFailures(
+ std::string path,
+ std::string op_name)
+{
+ auto op =getOpCode(op_name);
+ if (op >= NUM_OP_CODES) {
+ return NULL;
+ }
+
+ auto iter = mAllInjectedFailures_.find(path);
+ if (iter != mAllInjectedFailures_.end()) {
+ FilePathFailures* failure = iter->second;
+ return failure->GetFailures(op);
+ }
+ return NULL;
+}
+
+void FailureInjector::InjectFailure(
+ std::string path,
+ std::string op_name,
+ InjectedAction action)
+{
+ InjectFailure(path, getOpCode(op_name), action);
+}
+
+void FailureInjector::InjectFailure(
+ std::string path,
+ InjectedOpCode op,
+ InjectedAction action)
+{
+ if (op >= NUM_OP_CODES) {
+ return;
+ }
+ auto iter = mAllInjectedFailures_.find(path);
+ if (iter == mAllInjectedFailures_.end()) {
+ FilePathFailures* failure = new FilePathFailures();
+ failure->addFailure(op, action);
+ mAllInjectedFailures_[path] = failure;
+ } else {
+ FilePathFailures* failure = iter->second;
+ failure->addFailure(op, action);
+ }
+
+}
+
+void FailureInjector::ResetFailure(
+ std::string path,
+ std::string op_name)
+{
+ auto op =getOpCode(op_name);
+ if (op >= NUM_OP_CODES) {
+ return NULL;
+ }
+ ResetFailure(path, op);
+}
+
+void FailureInjector::ResetFailure(
+ std::string path,
+ InjectedOpCode op)
+{
+ auto iter = mAllInjectedFailures_.find(path);
+ if (iter != mAllInjectedFailures_.end()) {
+ FilePathFailures* failure = iter->second;
+ failure->resetFailure(op);
+ }
+}
+
+void FailureInjector::ResetFailure(
+ std::string path)
+{
+ auto iter = mAllInjectedFailures_.find(path);
+ if (iter != mAllInjectedFailures_.end()) {
+ FilePathFailures* failure = iter->second;
+ failure->resetFailure();
+ mAllInjectedFailures_.erase(path);
+ }
+}
+
+void FailureInjector::ResetFailure()
+{
+ for (auto iter : mAllInjectedFailures_) {
+ FilePathFailures* failure = iter.second;
+ failure->resetFailure();
+ }
+ mAllInjectedFailures_.clear();
+}
+
+void FailureInjector::dumpFailures(std::string path)
+{
+ auto iter = mAllInjectedFailures_.find(path);
+ if (iter != mAllInjectedFailures_.end()) {
+ FilePathFailures* failure = iter->second;
+ failure->dumpFailures();
+ }
+}
+
+void FailureInjector::dumpFailures()
+{
+ for (auto iter : mAllInjectedFailures_) {
+ FilePathFailures* failure = iter.second;
+ cout << "Path: " << iter.first << "\n";
+ failure->dumpFailures();
+ }
+}
diff --git a/tools/fault-injection-service/FileSystem/failure_injector.h
b/tools/fault-injection-service/FileSystem/failure_injector.h
new file mode 100644
index 0000000..8cce0df
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/failure_injector.h
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#ifndef __FAILURE_INJECTOR_H__
+#define __FAILURE_INJECTOR_H__
+
+#include<string>
+#include<map>
+#include<vector>
+
+#define CORRUPT_DATA_ERROR_CODE (99999)
+#define MICROSECONDS_IN_A_SECOND (1000 * 1000)
+
+#define InjectedOperations(GEN_CODE) \
+ GEN_CODE(GETATTR), \
+ GEN_CODE(READLINK), \
+ GEN_CODE(MKNOD), \
+ GEN_CODE(MKDIR), \
+ GEN_CODE(UNLINK), \
+ GEN_CODE(RMDIR), \
+ GEN_CODE(SYMLINK), \
+ GEN_CODE(RENAME), \
+ GEN_CODE(LINK), \
+ GEN_CODE(CHMOD), \
+ GEN_CODE(CHOWN), \
+ GEN_CODE(TRUNCATE), \
+ GEN_CODE(OPEN), \
+ GEN_CODE(READ), \
+ GEN_CODE(WRITE), \
+ GEN_CODE(STATFS), \
+ GEN_CODE(FLUSH), \
+ GEN_CODE(RELEASE), \
+ GEN_CODE(FSYNC), \
+ GEN_CODE(SETXATTR), \
+ GEN_CODE(GETXATTR), \
+ GEN_CODE(LISTXATTR), \
+ GEN_CODE(REMOVEXATTR), \
+ GEN_CODE(OPENDIR), \
+ GEN_CODE(READDIR), \
+ GEN_CODE(RELEASEDIR), \
+ GEN_CODE(FSYNCDIR), \
+ GEN_CODE(INIT), \
+ GEN_CODE(DESTROY), \
+ GEN_CODE(ACCESS), \
+ GEN_CODE(CREATE), \
+ GEN_CODE(LOCK), \
+ GEN_CODE(UTIMENS), \
+ GEN_CODE(BMAP), \
+ GEN_CODE(IOCTL), \
+ GEN_CODE(POLL), \
+ GEN_CODE(WRITE_BUF), \
+ GEN_CODE(READ_BUF), \
+ GEN_CODE(FLOCK), \
+ GEN_CODE(FALLOCATE), \
+ GEN_CODE(COPY_FILE_RANGE), \
+ GEN_CODE(LSEEK), \
+ GEN_CODE(NUM_OP_CODES)
+
+#define GEN_CODE_ENUM(x) x
+
+namespace NoiseInjector {
+
+enum InjectedOpCode {NO_OP = 0, InjectedOperations(GEN_CODE_ENUM)};
+
+struct InjectedAction {
+ enum ActionCode {DELAY = 0, FAIL, CORRUPT} code;
+ int delay; /* In seconds */
+ int error_code;
+
+ InjectedAction(int c, int d, int e) : delay(d), error_code(e) {
+ switch (c) {
+ case DELAY:
+ code = DELAY;
+ break;
+ case FAIL:
+ code = FAIL;
+ break;
+ case CORRUPT:
+ code = CORRUPT;
+ break;
+ }
+ };
+};
+
+class FilePathFailures {
+public:
+ std::vector<InjectedAction>* GetFailures(InjectedOpCode op);
+ void addFailure(InjectedOpCode op, InjectedAction action);
+ void resetFailure(InjectedOpCode op);
+ void resetFailure();
+ void dumpFailures();
+
+private:
+ /*
+ * Same file path could have multiple failures injected for
+ * different operations. For each {path, operation}, it
+ * is possible to have more than on action. For example,
+ * path {/opt/ABC/, read} injected failures could be
+ * 1. Delay the operation for 50 miliseconds, followed by
+ * 2. Fail the operation with EIO
+ * simulating a malfunction disk.
+ */
+
+ std::map<InjectedOpCode, std::vector<InjectedAction> > mInjectedOperations;
+};
+
+class FailureInjector {
+public:
+ std::vector<InjectedAction>* GetFailures(std::string path,
+ std::string op_name);
+ void InjectFailure(std::string path, std::string op_name,
+ InjectedAction action);
+ void InjectFailure(std::string path, InjectedOpCode op,
+ InjectedAction action);
+ void ResetFailure(std::string path, std::string op_name);
+ void ResetFailure(std::string path, InjectedOpCode op);
+ void ResetFailure(std::string path);
+ void ResetFailure();
+ void dumpFailures(std::string path);
+ void dumpFailures();
+ InjectedOpCode getOpCode(std::string op_name);
+
+private:
+ std::map<std::string, FilePathFailures *> mAllInjectedFailures_;
+
+friend class TestFailureInjector;
+};
+
+} /* namespace NoiseInjector */
+
+#endif /* __FAILURE_INJECTOR_H */
diff --git a/tools/fault-injection-service/FileSystem/failure_injector_fs.cc
b/tools/fault-injection-service/FileSystem/failure_injector_fs.cc
new file mode 100644
index 0000000..ad03e74
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/failure_injector_fs.cc
@@ -0,0 +1,912 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+
+#define FUSE_USE_VERSION 31
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef linux
+/* For pread()/pwrite()/utimensat() */
+#define _XOPEN_SOURCE 700
+#endif
+
+#include <libgen.h>
+#include <stdlib.h>
+
+#include <fuse3/fuse.h>
+#include <stdio.h>
+#include <ctime>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <errno.h>
+#ifdef __FreeBSD__
+#include <sys/socket.h>
+#include <sys/un.h>
+#endif
+#include <sys/time.h>
+
+
+#ifdef HAVE_SETXATTR
+#include <sys/xattr.h>
+#endif
+#include "failure_injector_fs.h"
+#include "failure_injector.h"
+#include "run_grpc_service.h"
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <strings.h>
+#include <thread>
+#include <cstdio>
+
+using NoiseInjector::FailureInjector;
+using NoiseInjector::RunGrpcService;
+
+using namespace std;
+using namespace NoiseInjector::FileSystem;
+
+#define LOGFILE "/var/log/noise_injector_log.txt"
+
+struct fuse_operations FailureInjectorFs::mFuseOperationsVec;
+FailureInjector *FailureInjectorFs::mFailureInjector = NULL;
+RunGrpcService *FailureInjectorFs::mGrpcSever = NULL;
+
+FailureInjectorFs::FailureInjectorFs(FailureInjector *injector)
+{
+ mFailureInjector = injector;
+ mGrpcSever = new RunGrpcService(mFailureInjector);
+}
+
+int FailureInjectorFs::mknod_wrapper(
+ int dirfd,
+ const char *path,
+ const char *link,
+ int mode,
+ dev_t rdev)
+{
+ int res;
+
+ if (S_ISREG(mode)) {
+ res = openat(dirfd, path, O_CREAT | O_EXCL | O_WRONLY, mode);
+ if (res >= 0)
+ res = close(res);
+ } else if (S_ISDIR(mode)) {
+ res = mkdirat(dirfd, path, mode);
+ } else if (S_ISLNK(mode) && link != NULL) {
+ res = symlinkat(link, dirfd, path);
+ } else if (S_ISFIFO(mode)) {
+ res = mkfifoat(dirfd, path, mode);
+ } else {
+ res = mknodat(dirfd, path, mode, rdev);
+ }
+
+ return res;
+}
+
+void *FailureInjectorFs::fifs_init(
+ struct fuse_conn_info *conn,
+ struct fuse_config *cfg)
+{
+ extern void RunServer();
+ (void) conn;
+ cfg->use_ino = 1;
+
+ std::freopen(LOGFILE, "w", stdout);
+
+ /*
+ * Pick up changes from lower filesystem right away. This is
+ * also necessary for better hardlink support. When the kernel
+ * calls the unlink() handler, it does not know the inode of
+ * the to-be-removed entry and can therefore not invalidate
+ * the cache of the associated inode - resulting in an
+ * incorrect st_nlink value being reported for any remaining
+ * hardlinks to this inode.
+ */
+ cfg->entry_timeout = 0;
+ cfg->attr_timeout = 0;
+ cfg->negative_timeout = 0;
+
+ /* Create a separete GRPC server thread to accept failure injection. */
+ std::thread th(&RunGrpcService::RunServer, mGrpcSever);
+ th.detach();
+
+ return NULL;
+}
+
+int FailureInjectorFs::fifs_getattr(
+ const char *path,
+ struct stat *stbuf,
+ struct fuse_file_info *fi)
+{
+ (void) fi;
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"GETATTR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = lstat(path, stbuf);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_access(
+ const char *path,
+ int mask)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"ACCESS", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = access(path, mask);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_readlink(
+ const char *path,
+ char *buf,
+ size_t size)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"READLINK", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = readlink(path, buf, size - 1);
+ if (res == -1)
+ return -errno;
+
+ buf[res] = '\0';
+ return 0;
+}
+
+int FailureInjectorFs::fifs_readdir(
+ const char *path,
+ void *buf,
+ fuse_fill_dir_t filler,
+ off_t offset,
+ struct fuse_file_info *fi,
+ enum fuse_readdir_flags flags)
+{
+ DIR *dp;
+ struct dirent *de;
+ int injected_error = 0;
+
+ (void) offset;
+ (void) fi;
+ (void) flags;
+
+ if (CheckForInjectedError(path,"READDIR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ dp = opendir(path);
+ if (dp == NULL)
+ return -errno;
+
+ while ((de = readdir(dp)) != NULL) {
+ struct stat st;
+ memset(&st, 0, sizeof(st));
+ st.st_ino = de->d_ino;
+ st.st_mode = de->d_type << 12;
+ if (filler(buf, de->d_name, &st, 0, (fuse_fill_dir_flags)0))
+ break;
+ }
+
+ closedir(dp);
+ return 0;
+}
+
+int FailureInjectorFs::fifs_mknod(
+ const char *path,
+ mode_t mode,
+ dev_t rdev)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"MKNOD", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = mknod_wrapper(AT_FDCWD, path, NULL, mode, rdev);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_mkdir(
+ const char *path,
+ mode_t mode)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"MKDIR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = mkdir(path, mode);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_unlink(const char *path)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"UNLINK", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = unlink(path);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_rmdir(const char *path)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"RMDIR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = rmdir(path);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_symlink(
+ const char *from,
+ const char *to)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(from, "SYMLINK", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = symlink(from, to);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_rename(
+ const char *from,
+ const char *to,
+ unsigned int flags)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(from, "RENAME", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ if (flags)
+ return -EINVAL;
+
+ res = rename(from, to);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_link(
+ const char *from,
+ const char *to)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(from,"LINK", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = link(from, to);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_chmod(
+ const char *path,
+ mode_t mode,
+ struct fuse_file_info *fi)
+{
+ (void) fi;
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"CHMOD", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = chmod(path, mode);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_chown(
+ const char *path,
+ uid_t uid,
+ gid_t gid,
+ struct fuse_file_info *fi)
+{
+ (void) fi;
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"CHOWN", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = lchown(path, uid, gid);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_truncate(
+ const char *path,
+ off_t size,
+ struct fuse_file_info *fi)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"TRUNCATE", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ if (fi != NULL)
+ res = ftruncate(fi->fh, size);
+ else
+ res = truncate(path, size);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+#ifdef HAVE_UTIMENSAT
+int FailureInjectorFs::fifs_utimens(
+ const char *path,
+ const struct timespec ts[2],
+ struct fuse_file_info *fi)
+{
+ (void) fi;
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"UTIMENS", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+
+ /* don't use utime/utimes since they follow symlinks */
+ res = utimensat(0, path, ts, AT_SYMLINK_NOFOLLOW);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+#endif
+
+int FailureInjectorFs::fifs_create(
+ const char *path,
+ mode_t mode,
+ struct fuse_file_info *fi)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"CREATE", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = open(path, fi->flags, mode);
+ if (res == -1)
+ return -errno;
+
+ fi->fh = res;
+ return 0;
+}
+
+int FailureInjectorFs::fifs_open(
+ const char *path,
+ struct fuse_file_info *fi)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"OPEN", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = open(path, fi->flags);
+ if (res == -1)
+ return -errno;
+
+ fi->fh = res;
+ return 0;
+}
+
+bool FailureInjectorFs::CheckForInjectedError(
+ string path,
+ string op,
+ int *injected_error)
+{
+ char *mpath = strdup(path.c_str());
+ if (mFailureInjector == NULL) {
+ return false;
+ }
+
+ auto failures = mFailureInjector->GetFailures(path, op);
+ while (failures == NULL) {
+ // Check if failures are injected in any of the parent directories.
+ char *dir = dirname(mpath);
+ failures = mFailureInjector->GetFailures(string(dir), op);
+ if ((failures == NULL) && (!strcmp(dir, ".") || !strcmp(dir, "/"))) {
+ free(mpath);
+ return false;
+ }
+ mpath = dir;
+ }
+ free(mpath);
+
+ *injected_error = 0;
+ useconds_t delay = 1;
+ for (auto f : *failures) {
+ switch(f.code) {
+ case InjectedAction::DELAY:
+ delay = f.delay * MICROSECONDS_IN_A_SECOND;
+ break;
+ case InjectedAction::FAIL:
+ if (*injected_error != CORRUPT_DATA_ERROR_CODE) {
+ *injected_error = f.error_code;
+ }
+ break;
+ case InjectedAction::CORRUPT:
+ *injected_error = CORRUPT_DATA_ERROR_CODE;
+ break;
+ default:
+ // Ignore for now.
+ break;
+ }
+ }
+ // First create the delay.
+ usleep(delay);
+
+ if (*injected_error) {
+ return true;
+ }
+ return false;
+}
+
+void FailureInjectorFs::FillCorruptData(
+ char *buf,
+ size_t size)
+{
+ // For now just fill with some pattern based on input.
+ // TBD : Fill with random data pattern.
+ for (size_t i = 0; i < size; ++ i) {
+ buf[i] = buf[i] + 2;
+ }
+}
+
+int FailureInjectorFs::fifs_read(
+ const char *path,
+ char *buf,
+ size_t size,
+ off_t offset,
+ struct fuse_file_info *fi)
+{
+ int fd;
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path, "READ", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ if (injected_error != CORRUPT_DATA_ERROR_CODE) {
+ return -injected_error;
+ }
+ }
+
+ if(fi == NULL)
+ fd = open(path, O_RDONLY);
+ else
+ fd = fi->fh;
+
+ if (fd == -1)
+ return -errno;
+
+ res = pread(fd, buf, size, offset);
+ if (res == -1)
+ res = -errno;
+
+ if(fi == NULL)
+ close(fd);
+ if ((res != -1) && (injected_error == CORRUPT_DATA_ERROR_CODE)) {
+ FillCorruptData(buf, size);
+ return res;
+ }
+ return res;
+}
+
+int FailureInjectorFs::fifs_write(
+ const char *path,
+ const char *buf,
+ size_t size,
+ off_t offset,
+ struct fuse_file_info *fi)
+{
+ int fd;
+ int res;
+ int injected_error = 0;
+ (void) fi;
+
+ if (CheckForInjectedError(path, "WRITE", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ if (injected_error != CORRUPT_DATA_ERROR_CODE) {
+ return -injected_error;
+ } else {
+ FillCorruptData(const_cast<char *>(buf), size);
+ }
+ }
+
+ if(fi == NULL)
+ fd = open(path, O_WRONLY);
+ else
+ fd = fi->fh;
+
+ if (fd == -1)
+ return -errno;
+
+ res = pwrite(fd, buf, size, offset);
+ if (res == -1)
+ res = -errno;
+
+ if(fi == NULL)
+ close(fd);
+ return res;
+}
+
+int FailureInjectorFs::fifs_statfs(
+ const char *path,
+ struct statvfs *stbuf)
+{
+ int res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path, "STATFS", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ res = statvfs(path, stbuf);
+ if (res == -1)
+ return -errno;
+
+ return 0;
+}
+
+int FailureInjectorFs::fifs_release(
+ const char *path,
+ struct fuse_file_info *fi)
+{
+ (void) path;
+ close(fi->fh);
+ return 0;
+}
+
+int FailureInjectorFs::fifs_fsync(
+ const char *path,
+ int isdatasync,
+ struct fuse_file_info *fi)
+{
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"FSYNC", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+
+ (void) path;
+ (void) isdatasync;
+ (void) fi;
+ return 0;
+}
+
+#ifdef HAVE_POSIX_FALLOCATE
+int FailureInjectorFs::fifs_fallocate(
+ const char *path,
+ int mode,
+ off_t offset,
+ off_t length,
+ struct fuse_file_info *fi)
+{
+ int fd;
+ int res;
+ int injected_error = 0;
+
+ (void) fi;
+
+ if (CheckForInjectedError(path,"FALLOCATE", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ if (mode)
+ return -EOPNOTSUPP;
+
+ if(fi == NULL)
+ fd = open(path, O_WRONLY);
+ else
+ fd = fi->fh;
+
+ if (fd == -1)
+ return -errno;
+
+ res = -posix_fallocate(fd, offset, length);
+
+ if(fi == NULL)
+ close(fd);
+ return res;
+}
+#endif
+
+#ifdef HAVE_SETXATTR
+/* xattr operations are optional and can safely be left unimplemented */
+int FailureInjectorFs::fifs_setxattr(
+ const char *path,
+ const char *name,
+ const char *value,
+ size_t size,
+ int flags)
+{
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"SETXATTR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ int res = lsetxattr(path, name, value, size, flags);
+ if (res == -1)
+ return -errno;
+ return 0;
+}
+
+int FailureInjectorFs::fifs_getxattr(
+ const char *path,
+ const char *name,
+ char *value,
+ size_t size)
+{
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"GETXATTR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ int res = lgetxattr(path, name, value, size);
+ if (res == -1)
+ return -errno;
+ return res;
+}
+
+int FailureInjectorFs::fifs_listxattr(
+ const char *path,
+ char *list,
+ size_t size)
+{
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"LISTXATTR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ int res = llistxattr(path, list, size);
+ if (res == -1)
+ return -errno;
+ return res;
+}
+
+int FailureInjectorFs::fifs_removexattr(
+ const char *path,
+ const char *name)
+{
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"REMOVEXATTR", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ int res = lremovexattr(path, name);
+ if (res == -1)
+ return -errno;
+ return 0;
+}
+#endif /* HAVE_SETXATTR */
+
+off_t FailureInjectorFs::fifs_lseek(
+ const char *path,
+ off_t off,
+ int whence,
+ struct fuse_file_info *fi)
+{
+ int fd;
+ off_t res;
+ int injected_error = 0;
+
+ if (CheckForInjectedError(path,"LSEEK", &injected_error)) {
+ cout << "Returning Injected error " << injected_error << "\n";
+ return -injected_error;
+ }
+
+ if (fi == NULL)
+ fd = open(path, O_RDONLY);
+ else
+ fd = fi->fh;
+
+ if (fd == -1)
+ return -errno;
+
+ res = lseek(fd, off, whence);
+ if (res == -1)
+ res = -errno;
+
+ if (fi == NULL)
+ close(fd);
+ return res;
+}
+
+void FailureInjectorFs::load_operations() {
+ mFuseOperationsVec.getattr = FailureInjectorFs::fifs_getattr;
+ mFuseOperationsVec.readlink = FailureInjectorFs::fifs_readlink;
+ mFuseOperationsVec.mknod = FailureInjectorFs::fifs_mknod;
+ mFuseOperationsVec.mkdir = FailureInjectorFs::fifs_mkdir;
+ mFuseOperationsVec.unlink = FailureInjectorFs::fifs_unlink;
+ mFuseOperationsVec.rmdir = FailureInjectorFs::fifs_rmdir;
+ mFuseOperationsVec.symlink = FailureInjectorFs::fifs_symlink;
+ mFuseOperationsVec.rename = FailureInjectorFs::fifs_rename;
+ mFuseOperationsVec.link = FailureInjectorFs::fifs_link;
+ mFuseOperationsVec.chmod = FailureInjectorFs::fifs_chmod;
+ mFuseOperationsVec.chown = FailureInjectorFs::fifs_chown;
+ mFuseOperationsVec.truncate = FailureInjectorFs::fifs_truncate;
+ mFuseOperationsVec.open = FailureInjectorFs::fifs_open;
+ mFuseOperationsVec.read = FailureInjectorFs::fifs_read;
+ mFuseOperationsVec.write = FailureInjectorFs::fifs_write;
+ mFuseOperationsVec.statfs = FailureInjectorFs::fifs_statfs;
+ mFuseOperationsVec.flush = NULL;
+ mFuseOperationsVec.release = FailureInjectorFs::fifs_release;
+ mFuseOperationsVec.fsync = FailureInjectorFs::fifs_fsync;
+
+#ifdef HAVE_SETXATTR
+ mFuseOperationsVec.setxattr = FailureInjectorFs::fifs_setxattr;
+ mFuseOperationsVec.getxattr = FailureInjectorFs::fifs_getxattr;
+ mFuseOperationsVec.listxattr = FailureInjectorFs::fifs_listxattr;
+ mFuseOperationsVec.removexattr = FailureInjectorFs::fifs_removexattr;
+#else
+ mFuseOperationsVec.setxattr = NULL;
+ mFuseOperationsVec.getxattr = NULL;
+ mFuseOperationsVec.listxattr = NULL;
+ mFuseOperationsVec.removexattr = NULL;
+#endif
+
+ mFuseOperationsVec.opendir = NULL;
+ mFuseOperationsVec.readdir = FailureInjectorFs::fifs_readdir;
+ mFuseOperationsVec.releasedir = NULL;
+ mFuseOperationsVec.fsyncdir = NULL;
+ mFuseOperationsVec.init = FailureInjectorFs::fifs_init;
+ mFuseOperationsVec.destroy = NULL;
+ mFuseOperationsVec.access = FailureInjectorFs::fifs_access;
+ mFuseOperationsVec.create = FailureInjectorFs::fifs_create;
+ mFuseOperationsVec.lock = NULL;
+
+#ifdef HAVE_UTIMENSAT
+ mFuseOperationsVec.utimens = FailureInjectorFs::fifs_utimens;
+#else
+ mFuseOperationsVec.utimens = NULL;
+#endif
+
+ mFuseOperationsVec.bmap = NULL;
+ mFuseOperationsVec.ioctl = NULL;
+ mFuseOperationsVec.poll = NULL;
+ mFuseOperationsVec.write_buf = NULL;
+ mFuseOperationsVec.read_buf = NULL;
+ mFuseOperationsVec.flock = NULL;
+
+#ifdef HAVE_POSIX_FALLOCATE
+ mFuseOperationsVec.fallocate = FailureInjectorFs::fifs_fallocate;
+#else
+ mFuseOperationsVec.fallocate = NULL;
+#endif
+
+ mFuseOperationsVec.copy_file_range = NULL;
+}
+
+const struct fuse_operations *FailureInjectorFs::getOperations() {
+ return &mFuseOperationsVec;
+}
+
+int main(int argc, char *argv[])
+{
+ FailureInjector *injector = new FailureInjector();
+ FailureInjectorFs noise_fs(injector);
+
+ noise_fs.load_operations();
+ umask(0);
+ fuse_main(argc, argv, noise_fs.getOperations(), NULL);
+}
diff --git a/tools/fault-injection-service/FileSystem/failure_injector_fs.h
b/tools/fault-injection-service/FileSystem/failure_injector_fs.h
new file mode 100644
index 0000000..20237c9
--- /dev/null
+++ b/tools/fault-injection-service/FileSystem/failure_injector_fs.h
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#ifndef __FAILURE_INJECTOR_FS_H
+#define __FAILURE_INJECTOR_FS_H
+
+#define FUSE_USE_VERSION 31
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef linux
+/* For pread()/pwrite()/utimensat() */
+#define _XOPEN_SOURCE 700
+#endif
+
+#include <fuse3/fuse.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <errno.h>
+#ifdef __FreeBSD__
+#include <sys/socket.h>
+#include <sys/un.h>
+#endif
+#include <sys/time.h>
+
+
+#ifdef HAVE_SETXATTR
+#include <sys/xattr.h>
+#endif
+
+#include<string>
+#include"failure_injector.h"
+#include "run_grpc_service.h"
+
+namespace NoiseInjector {
+
+namespace FileSystem {
+
+class FailureInjectorFs {
+private:
+ /* Helper function for mknod operation */
+ static int mknod_wrapper(int dirfd, const char *path,
+ const char *link, int mode, dev_t rdev);
+public:
+ FailureInjectorFs(FailureInjector *);
+ /*
+ * Following operations correspond with their corresponding
+ * Fuse operations vector.
+ */
+
+ static void *fifs_init(struct fuse_conn_info *conn,
+ struct fuse_config *cfg);
+
+ static int fifs_getattr(const char *path, struct stat *stbuf,
+ struct fuse_file_info *fi);
+
+ static int fifs_access(const char *path, int mask);
+
+ static int fifs_readlink(const char *path, char *buf, size_t size);
+
+ static int fifs_readdir(const char *path, void *buf, fuse_fill_dir_t
filler,
+ off_t offset, struct fuse_file_info *fi,
+ enum fuse_readdir_flags flags);
+
+ static int fifs_mknod(const char *path, mode_t mode, dev_t rdev);
+
+ static int fifs_mkdir(const char *path, mode_t mode);
+
+ static int fifs_unlink(const char *path);
+
+ static int fifs_rmdir(const char *path);
+
+ static int fifs_symlink(const char *from, const char *to);
+
+ static int fifs_rename(const char *from, const char *to,
+ unsigned int flags);
+
+ static int fifs_link(const char *from, const char *to);
+
+ static int fifs_chmod(const char *path, mode_t mode,
+ struct fuse_file_info *fi);
+
+ static int fifs_chown(const char *path, uid_t uid, gid_t gid,
+ struct fuse_file_info *fi);
+
+ static int fifs_truncate(const char *path, off_t size,
+ struct fuse_file_info *fi);
+
+#ifdef HAVE_UTIMENSAT
+ static int fifs_utimens(const char *path, const struct timespec ts[2],
+ struct fuse_file_info *fi);
+#endif
+
+ static int fifs_create(const char *path, mode_t mode,
+ struct fuse_file_info *fi);
+
+ static int fifs_open(const char *path, struct fuse_file_info *fi);
+
+ static int fifs_read(const char *path, char *buf, size_t size,
+ off_t offset, struct fuse_file_info *fi);
+
+ static int fifs_write(const char *path, const char *buf, size_t size,
+ off_t offset, struct fuse_file_info *fi);
+
+ static int fifs_statfs(const char *path, struct statvfs *stbuf);
+
+ static int fifs_release(const char *path, struct fuse_file_info *fi);
+
+ static int fifs_fsync(const char *path, int isdatasync,
+ struct fuse_file_info *fi);
+
+#ifdef HAVE_POSIX_FALLOCATE
+ static int fifs_fallocate(const char *path, int mode, off_t offset,
+ off_t length, struct fuse_file_info *fi);
+#endif
+
+#ifdef HAVE_SETXATTR
+ static int fifs_setxattr(const char *path, const char *name,
+ const char *value, size_t size, int flags);
+
+ static int fifs_getxattr(const char *path, const char *name, char *value,
+ size_t size);
+
+ static int fifs_listxattr(const char *path, char *list, size_t size);
+
+ static int fifs_removexattr(const char *path, const char *name);
+#endif /* HAVE_SETXATTR */
+
+ static off_t fifs_lseek(const char *path, off_t off, int whence,
+ struct fuse_file_info *fi);
+
+ /* Helper functions to setup/access Fuse operations vector */
+
+ static bool CheckForInjectedError(std::string path, std::string op,
+ int *injected_error);
+
+ static void FillCorruptData(char *buf, size_t size);
+
+ void load_operations();
+
+ const struct fuse_operations *getOperations();
+
+ FailureInjector& GetFailureInjector();
+
+private:
+ /* Fuse operations vector */
+ static struct fuse_operations mFuseOperationsVec;
+
+ static FailureInjector *mFailureInjector;
+
+ static RunGrpcService *mGrpcSever;
+};
+}
+}
+
+#endif /* __FAILURE_INJECTOR_FS_H */
diff --git a/tools/fault-injection-service/README.md
b/tools/fault-injection-service/README.md
new file mode 100644
index 0000000..d988107
--- /dev/null
+++ b/tools/fault-injection-service/README.md
@@ -0,0 +1,56 @@
+Fault Injection Service
+========================
+
+About
+------
+Fault Injection Service is designed to validate Ozone under heavy stress and
+failed or failing system components. This service runs independently of Ozone
+and offers a client tool to inject errors.
+
+Development Status
+------------------
+Currently this service can inject errors on the IO path. The next step would be
+to add ability to inject similar errors on the network path as well.
+
+Supported Platforms
+-------------------
+Linux
+
+Dependencies
+-------------
+- libfuse3
+ yum install fuse3.x86_64 fuse3-devel.x86_64 fuse3-libs.x86_64
+- protobuf 2.5 or higher
+- grpc
+ built/installed from sources
+- cmake 3.14
+ built/installed from sources
+- cppunit & cppunit-devel
+
+Installation
+------------
+mkdir Build
+cd Build
+do 'cmake ..'
+do 'make'
+
+
+This will build following binaries:
+- failure_injector_svc_server
+ usage : failure_injector_svc_server /mnt
+ It will create a Fuse mountpoint on /mnt and also start a grpc server
+ to listen for failure injections on this mountpoint.
+
+- failure_injector_svc_client
+ This is a grpc client that can be used to inject failures on the /mnt
+ mountpoint above. Currently it supports,
+ - Injecting delays on various filesystem interfaces.
+ - Injecting a specific failure on a specific path for a specific operation.
+ - Simulate temporary or on-disk data corruption on IO path.
+ - Reseting specific or all the failures injected so far.
+
+- some unit test binaries
+
+Security implications
+---------------------
+TBD
diff --git
a/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.cc
b/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.cc
new file mode 100644
index 0000000..bdedbb3
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.cc
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <unistd.h>
+
+#include <grpcpp/grpcpp.h>
+
+#include "failure_injection_service.grpc.pb.h"
+#include "failure_injector_svc_client.h"
+#include "failure_injector.h"
+
+using grpc::Channel;
+using grpc::ClientContext;
+using grpc::Status;
+
+using namespace NoiseInjector;
+using namespace std;
+
+FailureInjectorSvcClient::FailureInjectorSvcClient(
+ shared_ptr<Channel> channel)
+ : stub_(FailureInjectorSvc::NewStub(channel)) {}
+
+string FailureInjectorSvcClient::GetStatus(const string& user)
+{
+ GetStatusRequest request;
+ request.set_name(user);
+
+ GetStatusReply reply;
+
+ ClientContext context;
+
+ Status status = stub_->GetStatus(&context, request, &reply);
+
+ if (status.ok()) {
+ return reply.message();
+ } else {
+ cout << status.error_code() << ": " << status.error_message()
+ << endl;
+ return "RPC failed";
+ }
+}
+
+void FailureInjectorSvcClient::InjectFailure(
+ string path,
+ string op,
+ int action,
+ int param)
+
+{
+ InjectFailureRequest request;
+
+ switch (action) {
+ case 0:
+ request.set_action_code(InjectFailureRequest::DELAY);
+ break;
+ case 1:
+ request.set_action_code(InjectFailureRequest::FAIL);
+ break;
+ case 2:
+ request.set_action_code(InjectFailureRequest::CORRUPT);
+ break;
+ }
+
+ request.set_path(path);
+ request.set_op_name(op);
+ request.set_action_param(param);
+
+
+ InjectFailureReply reply;
+
+ ClientContext context;
+
+ Status status = stub_->InjectFailure(&context, request, &reply);
+
+ if (status.ok()) {
+ cout << "RPC Success\n";
+ } else {
+ cout << status.error_code() << ": " << status.error_message()
+ << endl;
+ cout << "RPC failed\n";
+ }
+}
+
+void FailureInjectorSvcClient::ResetFailure(
+ string path,
+ string op)
+{
+ ResetFailureRequest request;
+
+ request.set_path(path);
+ request.set_op_name(op);
+
+ ResetFailureReply reply;
+
+ ClientContext context;
+
+ Status status = stub_->ResetFailure(&context, request, &reply);
+
+ if (status.ok()) {
+ cout << "RPC Success\n";
+ } else {
+ cout << status.error_code() << ": " << status.error_message()
+ << endl;
+ cout << "RPC failed\n";
+ }
+}
+
+void usage(char *prog)
+{
+ cout << prog << ": -irs -o operation -p path -a action -d delay ";
+ cout << "-e error_code\n" ;
+ cout << " -i: Inject failure\n";
+ cout << " -r: Reset failure \n";
+ cout << " -s: status of NoiseInjection service \n";
+ cout << " -p path: path to inject failure \n";
+ cout << " -o operation: operation to inject failure \n";
+ cout << " -a action: injected action (0: DELAY/1: FAIL/2: CORRUPT) \n";
+ cout << " -d delay: delay in seconds\n";
+ cout << " -e error code to return\n";
+
+ exit(1);
+}
+
+int main(int argc, char** argv)
+{
+ // Instantiate the client.
+ FailureInjectorSvcClient injector(grpc::CreateChannel(
+ "localhost:50051", grpc::InsecureChannelCredentials()));
+ string reply;
+ int opt;
+ int flag = 0;
+ string path;
+ string operation;
+ int action = InjectedAction::FAIL;
+#define INVALID_PARAM (-9999)
+ int param = INVALID_PARAM;
+
+ while ((opt = getopt(argc, argv, "irsp:o:a:d:e:")) != -1) {
+ switch(opt) {
+ case 'i':
+ flag = 'i';
+ break;
+ case 'r':
+ flag = 'r';
+ break;
+ case 's':
+ flag = 's';
+ break;
+ case 'p':
+ path = optarg;
+ break;
+ case 'o':
+ operation = optarg;
+ break;
+ case 'a':
+ action =atoi(optarg);
+ break;
+ case 'd':
+ case 'e':
+ if (param == INVALID_PARAM) {
+ param =atoi(optarg);
+ } else {
+ usage(argv[0]);
+ }
+ break;
+ default:
+ usage(argv[0]);
+ }
+ }
+
+ switch (flag) {
+ case 'i':
+ if (param == INVALID_PARAM || action > InjectedAction::CORRUPT ||
+ path.empty() || operation.empty()) {
+ usage(argv[0]);
+ }
+ injector.InjectFailure(path, operation, action, param);
+ break;
+ case 'r':
+ injector.ResetFailure(path, operation);
+ break;
+ case 's':
+ reply = injector.GetStatus("Some Status");
+ cout << "FailureInjectorSvc received: ";
+ cout << reply << endl;
+ break;
+ default:
+ usage(argv[0]);
+ }
+
+ return 0;
+}
diff --git
a/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.h
b/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.h
new file mode 100644
index 0000000..3fccdbc
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/failure_injector_svc_client.h
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#ifndef __FAILURE_INJECTOR_SVC_CLIENT_H
+#define __FAILURE_INJECTOR_SVC_CLIENT_H
+
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include <grpcpp/grpcpp.h>
+
+#include "failure_injection_service.grpc.pb.h"
+
+using grpc::Channel;
+using grpc::ClientContext;
+using grpc::Status;
+
+namespace NoiseInjector {
+
+class FailureInjectorSvcClient {
+ public:
+ FailureInjectorSvcClient(std::shared_ptr<Channel> channel);
+
+ std::string GetStatus(const std::string& user);
+
+ void InjectFailure(std::string path, std::string op, int action, int param);
+
+ void ResetFailure(std::string path, std::string op);
+
+ private:
+ std::unique_ptr<FailureInjectorSvc::Stub> stub_;
+};
+
+}
+
+#endif /* __FAILURE_INJECTOR_SVC_CLIENT_H */
diff --git
a/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.cc
b/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.cc
new file mode 100644
index 0000000..ac7919c
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.cc
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include <grpcpp/grpcpp.h>
+
+#include "failure_injection_service.grpc.pb.h"
+#include "failure_injector.h"
+#include "failure_injector_svc_server.h"
+#include "run_grpc_service.h"
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::Status;
+
+using namespace NoiseInjector;
+using namespace std;
+
+Status FailureInjectorSvcServiceImpl::GetStatus(
+ ServerContext* context,
+ const GetStatusRequest* request,
+ GetStatusReply* reply)
+{
+
+ string prefix("GetStatus ");
+ cout << "Recived request " << request->name() << "\n";
+ reply->set_message(prefix + request->name());
+ return Status::OK;
+}
+
+Status FailureInjectorSvcServiceImpl::InjectFailure(
+ ServerContext* context,
+ const InjectFailureRequest* request,
+ InjectFailureReply* reply)
+{
+ cout << "Recived request " << request->path() << ":"
+ << request->op_name() << "\n";
+ mFailureInjector->InjectFailure(request->path(), request->op_name(),
+ InjectedAction((int)request->action_code(),
+ (int)request->action_param(),
+ (int)request->action_param()));
+ return Status::OK;
+}
+
+Status FailureInjectorSvcServiceImpl::ResetFailure(
+ ServerContext* context,
+ const ResetFailureRequest* request,
+ ResetFailureReply* reply)
+{
+ cout << "Recived request ";
+ if (request->has_path()) {
+ cout << request->path() << "\n";
+ if (request->has_op_name()) {
+ cout << request->op_name() << "\n";
+ }
+ }
+
+ if (request->has_path() && !request->path().empty()) {
+ if (request->has_op_name() && !request->op_name().empty()) {
+ mFailureInjector->ResetFailure(request->path(),
request->op_name());
+ } else {
+ mFailureInjector->ResetFailure(request->path());
+ }
+ } else {
+ mFailureInjector->ResetFailure();
+ }
+ return Status::OK;
+}
+
+FailureInjectorSvcServiceImpl::FailureInjectorSvcServiceImpl(
+ FailureInjector* injector)
+{
+ mFailureInjector = injector;
+}
diff --git
a/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.h
b/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.h
new file mode 100644
index 0000000..369a5cb
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/failure_injector_svc_server.h
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#ifndef __FAILURE_INJECTOR_SVC_SERVER_H
+#define __FAILURE_INJECTOR_SVC_SERVER_H
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include <grpcpp/grpcpp.h>
+
+#include "failure_injection_service.grpc.pb.h"
+#include "failure_injector.h"
+#include "run_grpc_service.h"
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::Status;
+
+namespace NoiseInjector {
+
+class FailureInjectorSvcServiceImpl final :
+ public FailureInjectorSvc::Service {
+
+public:
+ Status GetStatus(ServerContext* context,
+ const GetStatusRequest* request,
+ GetStatusReply* reply) override;
+
+ Status InjectFailure(ServerContext* context,
+ const InjectFailureRequest* request,
+ InjectFailureReply* reply) override;
+
+ Status ResetFailure(ServerContext* context,
+ const ResetFailureRequest* request,
+ ResetFailureReply* reply);
+
+ FailureInjectorSvcServiceImpl(FailureInjector* injector);
+
+private :
+ FailureInjector *mFailureInjector;
+};
+
+}
+
+#endif /* __FAILURE_INJECTOR_SVC_SERVER_H */
diff --git a/tools/fault-injection-service/Service/cpp/run_grpc_service.cc
b/tools/fault-injection-service/Service/cpp/run_grpc_service.cc
new file mode 100644
index 0000000..b3d88b9
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/run_grpc_service.cc
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include <grpcpp/grpcpp.h>
+
+#include "failure_injection_service.grpc.pb.h"
+#include "failure_injector_svc_server.h"
+#include "run_grpc_service.h"
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+
+using namespace NoiseInjector;
+using namespace std;
+
+void RunGrpcService::RunServer()
+{
+ string server_address("0.0.0.0:50051");
+ FailureInjectorSvcServiceImpl service(mFailureInjector);
+
+ ServerBuilder builder;
+ // Listen on the given address without any authentication mechanism.
+ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
+ // Register "service" as the instance through which we'll communicate with
+ // clients. In this case it corresponds to an *synchronous* service.
+ builder.RegisterService(&service);
+ // Finally assemble the server.
+ unique_ptr<Server> server(builder.BuildAndStart());
+ cout << "Server listening on " << server_address << endl;
+
+ // Wait for the server to shutdown.
+ server->Wait();
+}
+
+RunGrpcService::RunGrpcService(FailureInjector *injector)
+{
+ mFailureInjector = injector;
+}
diff --git a/tools/fault-injection-service/Service/cpp/run_grpc_service.h
b/tools/fault-injection-service/Service/cpp/run_grpc_service.h
new file mode 100644
index 0000000..73a71d1
--- /dev/null
+++ b/tools/fault-injection-service/Service/cpp/run_grpc_service.h
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+#ifndef __RUN_GRPC_SERVICE_H__
+#define __RUN_GRPC_SERVICE_H__
+
+#include "failure_injector.h"
+
+namespace NoiseInjector {
+
+class RunGrpcService {
+public:
+ void RunServer();
+ RunGrpcService(FailureInjector *injector);
+
+private :
+ FailureInjector *mFailureInjector;
+};
+
+}
+
+#endif /* __RUN_GRPC_SERVICE_H__ */
diff --git
a/tools/fault-injection-service/Service/protos/failure_injection_service.proto
b/tools/fault-injection-service/Service/protos/failure_injection_service.proto
new file mode 100644
index 0000000..1886621
--- /dev/null
+++
b/tools/fault-injection-service/Service/protos/failure_injection_service.proto
@@ -0,0 +1,68 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with this
+// work for additional information regarding copyright ownership. The ASF
+// licenses this file to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance with the
License.
+// You may obtain a copy of the License at
+// <p>
+// http://www.apache.org/licenses/LICENSE-2.0
+// <p>
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
under
+// the License.
+//
+
+syntax = "proto2";
+
+package NoiseInjector;
+
+// The failure injection service definition.
+service FailureInjectorSvc {
+ // Get Service Status
+ rpc GetStatus (GetStatusRequest) returns (GetStatusReply) {
+ }
+
+ // Inject Failure
+ rpc InjectFailure (InjectFailureRequest) returns (InjectFailureReply) {
+ }
+
+ // Reset Failure
+ rpc ResetFailure (ResetFailureRequest) returns (ResetFailureReply) {
+ }
+}
+
+// The request message containing the user's name.
+message GetStatusRequest {
+ optional string name = 1;
+}
+
+// The response message containing the greetings
+message GetStatusReply {
+ optional string message = 1;
+}
+
+message InjectFailureRequest {
+ required string path = 1;
+ required string op_name = 2;
+ enum ActionCode {
+ DELAY = 0;
+ FAIL = 1;
+ CORRUPT = 2;
+ };
+ required ActionCode action_code = 3;
+ optional int64 action_param = 4;
+};
+
+message InjectFailureReply {
+}
+
+message ResetFailureRequest {
+ optional string path = 1;
+ optional string op_name = 2;
+};
+
+message ResetFailureReply {
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
