This is an automated email from the ASF dual-hosted git repository.
mochen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 98bb0431b7 Persist bad disks (#10960)
98bb0431b7 is described below
commit 98bb0431b79ffeefe831e6d831458cccd8410ecb
Author: Mo Chen <[email protected]>
AuthorDate: Wed Jan 17 13:11:33 2024 -0600
Persist bad disks (#10960)
Add a config option to have traffic server remember which disks are bad.
Previously, cache disks that were determined to be bad at runtime were
forgotten upon restart. This could cause corrupt cache metadata to be loaded
from disk, leading to crashes, or slowdowns when accessing the known bad disk.
If the option is turned on, bad disks are remembered until the
known_bad_disks file is edited or deleted. If the system administrator replaces
the disks, they need to perform this manual step. Because of this requirement,
this option is off by default.
---
doc/admin-guide/files/records.yaml.en.rst | 27 +++++++++++-
include/tscore/Filenames.h | 1 +
src/iocore/cache/Cache.cc | 72 +++++++++++++++++++++++++++++++
src/records/RecordsConfig.cc | 2 +
src/traffic_layout/info.cc | 1 +
5 files changed, 102 insertions(+), 1 deletion(-)
diff --git a/doc/admin-guide/files/records.yaml.en.rst
b/doc/admin-guide/files/records.yaml.en.rst
index 4dc2b8ae06..68a312aad2 100644
--- a/doc/admin-guide/files/records.yaml.en.rst
+++ b/doc/admin-guide/files/records.yaml.en.rst
@@ -2477,7 +2477,7 @@ Cache Control
write vector. For further details on cache write vectors, refer to the
developer documentation for :cpp:class:`CacheVC`.
-.. ts::cv:: CONFIG proxy.config.cache.mutex_retry_delay INT 2
+.. ts:cv:: CONFIG proxy.config.cache.mutex_retry_delay INT 2
:reloadable:
:units: milliseconds
@@ -2488,6 +2488,31 @@ Cache Control
used from the asynchronous IO threads when IO finishes and the ``CacheVC``
lock or stripe lock is
required.
+.. ts:cv:: CONFIG proxy.config.cache.max_disk_errors INT 5
+
+ Cache disks sometimes fail due to hardware problems. |TS| keeps count of
+ the number of times it encounters I/O errors when accessing each cache disk.
+ If the number of errors on a disk reaches this setting, |TS| removes that
+ disk from the cache.
+
+ Note that the count of errors is kept in memory, and is reset to zero when
+ |TS| is restarted. By default, |TS| will not remember which cache disk has
+ been removed in this way when it restarts. If you wish to change this
behavior
+ and prevent known bad disks from re-joining the cache upon restart, change
+ the setting :ts:cv:`proxy.config.cache.persist_bad_disks`.
+
+.. ts:cv:: CONFIG proxy.config.cache.persist_bad_disks INT 0
+
+ When enabled (``1``), |TS| will remember which cache disks have been
+ marked as failed through :ts:cv:`proxy.config.cache.max_disk_errors`,
+ even after a restart. A list of known bad cache disks is written to
+ ``localstatedir/known_bad_disks.txt``. If you replace the known bad disks,
+ delete this file so that |TS| will use them in the cache again.
+
+ When disabled (``0``), |TS| will ignore ``known_bad_disks.txt``.
+
+ This feature is disabled by default.
+
RAM Cache
=========
diff --git a/include/tscore/Filenames.h b/include/tscore/Filenames.h
index c65723ddb1..b6ff616cb8 100644
--- a/include/tscore/Filenames.h
+++ b/include/tscore/Filenames.h
@@ -47,6 +47,7 @@ namespace filename
// Various other file names
constexpr const char *RECORDS_STATS = "records.snap";
constexpr const char *HOST_RECORDS = "host_records.yaml";
+ constexpr const char *BAD_DISKS = "bad_disks.txt";
} // namespace filename
} // namespace ts
diff --git a/src/iocore/cache/Cache.cc b/src/iocore/cache/Cache.cc
index 622cb42a03..d05f76f5a5 100644
--- a/src/iocore/cache/Cache.cc
+++ b/src/iocore/cache/Cache.cc
@@ -27,6 +27,7 @@
#include "P_CacheTest.h"
#include "tscore/Filenames.h"
+#include "tscore/Layout.h"
#include "../../records/P_RecProcess.h"
@@ -35,6 +36,11 @@
#endif
#include <atomic>
+#include <unordered_set>
+#include <fstream>
+#include <string>
+#include <system_error>
+#include <filesystem>
constexpr ts::VersionNumber CACHE_DB_VERSION(CACHE_DB_MAJOR_VERSION,
CACHE_DB_MINOR_VERSION);
@@ -67,6 +73,7 @@ int cache_config_read_while_writer = 0;
int cache_config_mutex_retry_delay = 2;
int cache_read_while_writer_retry_delay = 50;
int cache_config_read_while_writer_max_retries = 10;
+int cache_config_persist_bad_disks = false;
// Globals
@@ -94,6 +101,7 @@ ClassAllocator<CacheEvacuateDocVC>
cacheEvacuateDocVConnectionAllocator("cacheEv
ClassAllocator<EvacuationBlock> evacuationBlockAllocator("evacuationBlock");
ClassAllocator<CacheRemoveCont> cacheRemoveContAllocator("cacheRemoveCont");
ClassAllocator<EvacuationKey> evacuationKeyAllocator("evacuationKey");
+std::unordered_set<std::string> known_bad_disks;
namespace
{
@@ -336,6 +344,15 @@ CacheProcessor::start_internal(int flags)
opts |= O_CREAT;
}
+ // Check if the disk is known to be bad.
+ if (cache_config_persist_bad_disks && !known_bad_disks.empty()) {
+ if (known_bad_disks.find(paths[gndisks]) != known_bad_disks.end()) {
+ Warning("%s is a known bad disk. Skipping.", paths[gndisks]);
+ Metrics::Gauge::increment(cache_rsb.span_offline);
+ continue;
+ }
+ }
+
#ifdef O_DIRECT
opts |= O_DIRECT;
#endif
@@ -984,6 +1001,32 @@ Cache::vol_initialized(bool result)
}
}
+static void
+persist_bad_disks()
+{
+ std::filesystem::path localstatedir{Layout::get()->localstatedir};
+ std::filesystem::path bad_disks_path{localstatedir /
ts::filename::BAD_DISKS};
+ std::error_code ec;
+ std::filesystem::create_directories(bad_disks_path.parent_path(), ec);
+ if (ec) {
+ Error("Error creating directory for bad disks file: %s (%s)",
bad_disks_path.c_str(), ec.message().c_str());
+ return;
+ }
+
+ std::fstream bad_disks_file{bad_disks_path.c_str(), bad_disks_file.out};
+ if (bad_disks_file.good()) {
+ for (const std::string &path : known_bad_disks) {
+ bad_disks_file << path << std::endl;
+ if (bad_disks_file.fail()) {
+ Error("Error writing known bad disks file: %s",
bad_disks_path.c_str());
+ break;
+ }
+ }
+ } else {
+ Error("Failed to open file to persist bad disks: %s",
bad_disks_path.c_str());
+ }
+}
+
/** Set the state of a disk programmatically.
*/
bool
@@ -1046,6 +1089,11 @@ CacheProcessor::mark_storage_offline(CacheDisk *d, ///<
Target disk
}
}
+ if (cache_config_persist_bad_disks) {
+ known_bad_disks.insert(d->path);
+ persist_bad_disks();
+ }
+
return zret;
}
@@ -1907,6 +1955,30 @@ ink_cache_init(ts::ModuleVersion v)
REC_ReadConfigInteger(cacheProcessor.wait_for_cache,
"proxy.config.http.wait_for_cache");
+ REC_EstablishStaticConfigInt32(cache_config_persist_bad_disks,
"proxy.config.cache.persist_bad_disks");
+ Debug("cache_init", "proxy.config.cache.persist_bad_disks = %d",
cache_config_persist_bad_disks);
+ if (cache_config_persist_bad_disks) {
+ std::filesystem::path localstatedir{Layout::get()->localstatedir};
+ std::filesystem::path bad_disks_path{localstatedir /
ts::filename::BAD_DISKS};
+ std::fstream bad_disks_file{bad_disks_path.c_str(), bad_disks_file.in};
+ if (bad_disks_file.good()) {
+ for (std::string line; std::getline(bad_disks_file, line);) {
+ if (bad_disks_file.fail()) {
+ Error("Failed while trying to read known bad disks file: %s",
bad_disks_path.c_str());
+ break;
+ }
+ if (!line.empty()) {
+ known_bad_disks.insert(std::move(line));
+ }
+ }
+ }
+ // not having a bad disks file is not an error.
+
+ unsigned long known_bad_count = known_bad_disks.size();
+ Warning("%lu previously known bad disks were recorded in %s. They will
not be added to the cache.", known_bad_count,
+ bad_disks_path.c_str());
+ }
+
Result result = theCacheStore.read_config();
if (result.failed()) {
Fatal("Failed to read cache configuration %s: %s", ts::filename::STORAGE,
result.message());
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
index 4173656c74..3e69ef6b11 100644
--- a/src/records/RecordsConfig.cc
+++ b/src/records/RecordsConfig.cc
@@ -74,6 +74,8 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.cache.max_disk_errors", RECD_INT, "5",
RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.cache.persist_bad_disks", RECD_INT, "0",
RECU_RESTART_TS, RR_NULL, RECC_NULL, "[01]", RECA_NULL}
+ ,
{RECT_CONFIG, "proxy.config.output.logfile.name", RECD_STRING,
"traffic.out", RECU_RESTART_TM, RR_REQUIRED, RECC_NULL, nullptr,
RECA_NULL}
,
diff --git a/src/traffic_layout/info.cc b/src/traffic_layout/info.cc
index 791329bff2..54b9a532f8 100644
--- a/src/traffic_layout/info.cc
+++ b/src/traffic_layout/info.cc
@@ -159,6 +159,7 @@ produce_layout(bool json)
print_var("RUNTIMEDIR", RecConfigReadRuntimeDir(), json);
print_var("PLUGINDIR", RecConfigReadPluginDir(), json);
print_var("INCLUDEDIR", Layout::get()->includedir, json);
+ print_var("LOCALSTATEDIR", Layout::get()->localstatedir, json);
print_var(ts::filename::RECORDS, RecConfigReadConfigPath(nullptr,
ts::filename::RECORDS), json);
print_var(ts::filename::REMAP,
RecConfigReadConfigPath("proxy.config.url_remap.filename"), json);
