This is an automated email from the ASF dual-hosted git repository. bcall pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit aa908d1ee7afc89c8a0f34ebd5932f6238b8cfb1 Author: Randall Meyer <[email protected]> AuthorDate: Mon May 7 13:26:06 2018 +0100 Converts logging.config and ssl_server_name.config to YAML-based files --- CMakeLists.txt | 5 +- iocore/net/LuaSNIConfig.cc | 127 -------------- iocore/net/LuaSNIConfig.h | 114 ------------- iocore/net/Makefile.am | 10 +- iocore/net/P_SSLSNI.h | 4 +- iocore/net/SSLSNIConfig.cc | 23 ++- iocore/net/YamlSNIConfig.cc | 107 ++++++++++++ iocore/net/YamlSNIConfig.h | 62 +++++++ lib/Makefile.am | 10 +- lib/ts/EnumDescriptor.h | 61 +++++++ proxy/config/logging.config.default | 234 ++++++++++---------------- proxy/logging/LogConfig.cc | 32 +++- proxy/logging/LogConfig.h | 4 + proxy/logging/LogField.cc | 4 +- proxy/logging/LogField.h | 2 +- proxy/logging/LogFilter.cc | 2 +- proxy/logging/LogFilter.h | 2 +- proxy/logging/Makefile.am | 5 +- proxy/logging/YamlLogConfig.cc | 247 ++++++++++++++++++++++++++++ proxy/logging/YamlLogConfig.h | 46 ++++++ proxy/logging/YamlLogConfigDecoders.cc | 118 +++++++++++++ proxy/logging/YamlLogConfigDecoders.h | 41 +++++ src/traffic_logcat/Makefile.inc | 3 +- src/traffic_logstats/Makefile.inc | 1 + src/traffic_server/Makefile.inc | 1 + tests/gold_tests/h2/httpbin.test.py | 15 +- tests/gold_tests/logging/ccid_ctid.test.py | 16 +- tests/gold_tests/logging/custom-log.test.py | 16 +- tests/gold_tests/logging/log-field.test.py | 13 +- 29 files changed, 866 insertions(+), 459 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f03d8a..c92624a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ include_directories(lib lib/records lib/wccp lib/cppapi/include + lib/yamlcpp/include iocore/eventsystem iocore/net iocore/dns @@ -276,8 +277,8 @@ add_executable(ats iocore/net/I_UDPNet.h iocore/net/I_UDPPacket.h iocore/net/Inline.cc - iocore/net/LuaSNIConfig.cc - iocore/net/LuaSNIConfig.h + iocore/net/YamlSNIConfig.cc + iocore/net/YamlSNIConfig.h iocore/net/Net.cc iocore/net/NetVConnection.cc iocore/net/NetVCTest.cc diff --git a/iocore/net/LuaSNIConfig.cc b/iocore/net/LuaSNIConfig.cc deleted file mode 100644 index 6a13157..0000000 --- a/iocore/net/LuaSNIConfig.cc +++ /dev/null @@ -1,127 +0,0 @@ -/** @file - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ - -#include "LuaSNIConfig.h" -#include <cstring> -#include "ts/Diags.h" -#include "P_SNIActionPerformer.h" -#include "tsconfig/Errata.h" -#include "tsconfig/TsConfigLua.h" - -TsConfigDescriptor LuaSNIConfig::desc = {TsConfigDescriptor::Type::ARRAY, "Array", "Item vector", "Vector"}; -TsConfigArrayDescriptor LuaSNIConfig::DESCRIPTOR(LuaSNIConfig::desc); -TsConfigDescriptor LuaSNIConfig::Item::FQDN_DESCRIPTOR = {TsConfigDescriptor::Type::STRING, "String", TS_fqdn, - "Fully Qualified Domain Name"}; -TsConfigDescriptor LuaSNIConfig::Item::DISABLE_h2_DESCRIPTOR = {TsConfigDescriptor::Type::BOOL, "Boolean", TS_disable_H2, - "Disable H2"}; -TsConfigEnumDescriptor LuaSNIConfig::Item::LEVEL_DESCRIPTOR = {TsConfigDescriptor::Type::ENUM, - "enum", - "Level", - "Level for client verification", - {{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; -TsConfigDescriptor LuaSNIConfig::Item::TUNNEL_DEST_DESCRIPTOR = {TsConfigDescriptor::Type::STRING, "String", TS_tunnel_route, - "tunnel route destination"}; -TsConfigDescriptor LuaSNIConfig::Item::CLIENT_CERT_DESCRIPTOR = {TsConfigDescriptor::Type::STRING, "String", TS_client_cert, - "Client certificate to present to the next hop server"}; -TsConfigDescriptor LuaSNIConfig::Item::VERIFY_NEXT_SERVER_DESCRIPTOR = {TsConfigDescriptor::Type::INT, "Int", - TS_verify_origin_server, "Next hop verification level"}; - -ts::Errata -LuaSNIConfig::loader(lua_State *L) -{ - ts::Errata zret; - // char buff[256]; - // int error; - - lua_getfield(L, LUA_GLOBALSINDEX, "server_config"); - int l_type = lua_type(L, -1); - - switch (l_type) { - case LUA_TTABLE: // this has to be a multidimensional table - lua_pushnil(L); - while (lua_next(L, -2) != 0) { - l_type = lua_type(L, -1); - if (l_type == LUA_TTABLE) { // the item should be table - // new Item - LuaSNIConfig::Item item; - item.loader(L); - items.push_back(item); - } else { - zret.push(ts::Errata::Message(0, 0, "Invalid Entry at SNI config")); - } - lua_pop(L, 1); - } - break; - case LUA_TSTRING: - Debug("ssl", "string value %s", lua_tostring(L, -1)); - break; - default: - zret.push(ts::Errata::Message(0, 0, "Invalid Lua SNI Config")); - Debug("ssl", "Please check your SNI config"); - break; - } - - return zret; -} - -ts::Errata -LuaSNIConfig::Item::loader(lua_State *L) -{ - ts::Errata zret; - //-1 will contain the subarray now (since it is a value in the main table)) - lua_pushnil(L); - while (lua_next(L, -2)) { - if (lua_type(L, -2) != LUA_TSTRING) { - Debug("ssl", "string keys expected for entries in %s", SSL_SERVER_NAME_CONFIG); - } - const char *name = lua_tostring(L, -2); - if (!strncmp(name, TS_fqdn, strlen(TS_fqdn))) { - FQDN_CONFIG.loader(L); - } else if (!strncmp(name, TS_disable_H2, strlen(TS_disable_H2))) { - DISABLEH2_CONFIG.loader(L); - } else if (!strncmp(name, TS_verify_client, strlen(TS_verify_client))) { - VERIFYCLIENT_CONFIG.loader(L); - } else if (!strncmp(name, TS_verify_origin_server, strlen(TS_verify_origin_server))) { - VERIFY_NEXT_SERVER_CONFIG.loader(L); - } else if (!strncmp(name, TS_client_cert, strlen(TS_client_cert))) { - CLIENT_CERT_CONFIG.loader(L); - } else if (!strncmp(name, TS_tunnel_route, strlen(TS_tunnel_route))) { - TUNNEL_DEST_CONFIG.loader(L); - } else { - zret.push(ts::Errata::Message(0, 0, "Invalid Entry at SNI config")); - } - lua_pop(L, 1); - } - return zret; -} - -ts::Errata -LuaSNIConfig::registerEnum(lua_State *L) -{ - ts::Errata zret; - lua_newtable(L); - lua_setglobal(L, "LevelTable"); - int i = start; - LUA_ENUM(L, "NONE", i++); - LUA_ENUM(L, "MODERATE", i++); - LUA_ENUM(L, "STRICT", i++); - return zret; -} diff --git a/iocore/net/LuaSNIConfig.h b/iocore/net/LuaSNIConfig.h deleted file mode 100644 index f301a6d..0000000 --- a/iocore/net/LuaSNIConfig.h +++ /dev/null @@ -1,114 +0,0 @@ -/** @file - - @section license License - - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -/* - * File: LuaSNIConfig.h - * Author: persia - * - * Created on September 21, 2017, 4:25 PM - */ - -#ifndef LUASNICONFIG_H -#define LUASNICONFIG_H - -#include "tsconfig/TsConfigLua.h" - -#include <vector> - -using ts::Errata; -#define LUA_ENUM(L, name, val) \ - lua_pushlstring(L, #name, sizeof(#name) - 1); \ - lua_pushnumber(L, val); \ - lua_settable(L, -3); - -constexpr char TS_fqdn[] = "fqdn"; -constexpr char TS_disable_H2[] = "disable_h2"; -constexpr char TS_verify_client[] = "verify_client"; -constexpr char TS_tunnel_route[] = "tunnel_route"; -constexpr char TS_verify_origin_server[] = "verify_origin_server"; -constexpr char TS_client_cert[] = "client_cert"; - -const int start = 0; -struct LuaSNIConfig : public TsConfigBase { - using self = LuaSNIConfig; - enum class Action { - disable_h2 = start, - verify_client, - tunnel_route, // blind tunnel action - verify_origin_server, // this applies to server side vc only - client_cert - - }; - enum class Level { NONE = 0, MODERATE, STRICT }; - static TsConfigDescriptor desc; - static TsConfigArrayDescriptor DESCRIPTOR; - - LuaSNIConfig() : TsConfigBase(this->DESCRIPTOR) { self::Item::Initialize(); } - - struct Item : public TsConfigBase { - Item() - : TsConfigBase(DESCRIPTOR), - FQDN_CONFIG(FQDN_DESCRIPTOR, fqdn), - DISABLEH2_CONFIG(DISABLE_h2_DESCRIPTOR, disable_h2), - VERIFYCLIENT_CONFIG(LEVEL_DESCRIPTOR, (int &)verify_client_level), - TUNNEL_DEST_CONFIG(TUNNEL_DEST_DESCRIPTOR, tunnel_destination), - CLIENT_CERT_CONFIG(CLIENT_CERT_DESCRIPTOR, client_cert), - VERIFY_NEXT_SERVER_CONFIG(VERIFY_NEXT_SERVER_DESCRIPTOR, (int &)verify_origin_server) - { - } - ts::Errata loader(lua_State *s) override; - static void - Initialize() - { - // ACTION_DESCRIPTOR.values = { - // {TS_disable_H2, 0}, {TS_verify_client, 1}, {TS_tunnel_route, 2}, {TS_verify_origin_server, 3}, {TS_client_cert, 4}}; - // - // ACTION_DESCRIPTOR.keys = { - // {0, TS_disable_H2}, {1, TS_verify_client}, {2, TS_tunnel_route}, {3, TS_verify_origin_server}, {4, TS_client_cert}}; - } - - std::string fqdn; - bool disable_h2 = false; - uint8_t verify_client_level = 0; - std::string tunnel_destination; - uint8_t verify_origin_server = 0; - std::string client_cert; - - // These need to be initialized statically. - static TsConfigObjectDescriptor OBJ_DESCRIPTOR; - static TsConfigDescriptor FQDN_DESCRIPTOR; - TsConfigString FQDN_CONFIG; - static TsConfigDescriptor DISABLE_h2_DESCRIPTOR; - TsConfigBool DISABLEH2_CONFIG; - static TsConfigEnumDescriptor LEVEL_DESCRIPTOR; - TsConfigEnum<self::Level> VERIFYCLIENT_CONFIG; - static TsConfigDescriptor TUNNEL_DEST_DESCRIPTOR; - TsConfigString TUNNEL_DEST_CONFIG; - static TsConfigDescriptor CLIENT_CERT_DESCRIPTOR; - TsConfigString CLIENT_CERT_CONFIG; - static TsConfigDescriptor VERIFY_NEXT_SERVER_DESCRIPTOR; - TsConfigInt VERIFY_NEXT_SERVER_CONFIG; - ~Item() override {} - }; - std::vector<self::Item> items; - ts::Errata loader(lua_State *s) override; - ts::Errata registerEnum(lua_State *L); -}; -#endif /* LUASNICONFIG_H */ diff --git a/iocore/net/Makefile.am b/iocore/net/Makefile.am index 6ebb1d5..ee66ec6 100644 --- a/iocore/net/Makefile.am +++ b/iocore/net/Makefile.am @@ -47,7 +47,8 @@ test_certlookup_LDADD = \ @OPENSSL_LIBS@ \ $(top_builddir)/lib/ts/libtsutil.la \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ - $(top_builddir)/proxy/ParentSelectionStrategy.o + $(top_builddir)/proxy/ParentSelectionStrategy.o \ + @LIB_YAMLCPP@ test_UDPNet_CPPFLAGS = \ $(AM_CPPFLAGS) \ @@ -73,8 +74,7 @@ test_UDPNet_LDADD = \ $(top_builddir)/lib/ts/libtsutil.la \ $(top_builddir)/proxy/ParentSelectionStrategy.o \ $(top_builddir)/lib/tsconfig/libtsconfig.la \ - $(top_builddir)/lib/luajit/src/libluajit.a \ - @LIBTCL@ @HWLOC_LIBS@ @OPENSSL_LIBS@ + @LIBTCL@ @HWLOC_LIBS@ @OPENSSL_LIBS@ @LIB_YAMLCPP@ test_UDPNet_SOURCES = \ test_I_UDPNet.cc @@ -93,8 +93,8 @@ libinknet_a_SOURCES = \ Inline.cc \ I_SessionAccept.h \ SessionAccept.cc \ - LuaSNIConfig.h \ - LuaSNIConfig.cc \ + YamlSNIConfig.h \ + YamlSNIConfig.cc \ Net.cc \ NetVConnection.cc \ P_SNIActionPerformer.h \ diff --git a/iocore/net/P_SSLSNI.h b/iocore/net/P_SSLSNI.h index a7f4463..06b7f2f 100644 --- a/iocore/net/P_SSLSNI.h +++ b/iocore/net/P_SSLSNI.h @@ -37,7 +37,7 @@ #include "openssl/ossl_typ.h" #include <vector> #include <strings.h> -#include "LuaSNIConfig.h" +#include "YamlSNIConfig.h" // Properties for the next hop server struct NextHopProperty { @@ -57,7 +57,7 @@ struct SNIConfigParams : public ConfigInfo { SNIMap wild_sni_action_map; NextHopPropertyTable next_hop_table; NextHopPropertyTable wild_next_hop_table; - LuaSNIConfig L_sni; + YamlSNIConfig Y_sni; NextHopProperty *getPropertyConfig(cchar *servername) const; SNIConfigParams(); ~SNIConfigParams() override; diff --git a/iocore/net/SSLSNIConfig.cc b/iocore/net/SSLSNIConfig.cc index 9c3cc5b..f140612 100644 --- a/iocore/net/SSLSNIConfig.cc +++ b/iocore/net/SSLSNIConfig.cc @@ -56,7 +56,7 @@ SNIConfigParams::getPropertyConfig(cchar *servername) const void SNIConfigParams::loadSNIConfig() { - for (auto item : L_sni.items) { + for (auto item : Y_sni.items) { actionVector *aiVec = new actionVector(); Debug("ssl", "name: %s", item.fqdn.data()); cchar *servername = item.fqdn.data(); @@ -139,28 +139,25 @@ int SNIConfigParams::Initialize() { sni_filename = ats_stringdup(RecConfigReadConfigPath("proxy.config.ssl.servername.filename")); + + Note("loading %s", sni_filename); + struct stat sbuf; if (stat(sni_filename, &sbuf) == -1 && errno == ENOENT) { + Note("failed to reload ssl_server_name.config"); Warning("Loading SNI configuration - filename: %s doesn't exist", sni_filename); return 1; } - lua_State *L = lua_open(); /* opens Lua */ - luaL_openlibs(L); - if (luaL_loadfile(L, sni_filename)) { - Error("Loading SNI configuration - luaL_loadfile: %s", lua_tostring(L, -1)); - lua_pop(L, 1); - return 1; - } - - if (lua_pcall(L, 0, 0, 0)) { - Error("Loading SNI configuration - luap_pcall: %s failed: %s", sni_filename, lua_tostring(L, -1)); - lua_pop(L, 1); + ts::Errata zret = Y_sni.loader(sni_filename); + if (!zret.isOK()) { + Note("failed to reload ssl_server_name.config"); return 1; } - L_sni.loader(L); loadSNIConfig(); + Note("ssl_server_name.config done reloading!"); + return 0; } diff --git a/iocore/net/YamlSNIConfig.cc b/iocore/net/YamlSNIConfig.cc new file mode 100644 index 0000000..5ab7ad2 --- /dev/null +++ b/iocore/net/YamlSNIConfig.cc @@ -0,0 +1,107 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "YamlSNIConfig.h" + +#include <unordered_map> +#include <set> +#include <string_view> + +#include <yaml-cpp/yaml.h> + +#include "ts/Diags.h" +#include "ts/EnumDescriptor.h" +#include "tsconfig/Errata.h" + +ts::Errata +YamlSNIConfig::loader(const char *cfgFilename) +{ + try { + YAML::Node config = YAML::LoadFile(cfgFilename); + if (!config.IsSequence()) { + return ts::Errata::Message(1, 1, "expected sequence"); + } + + for (auto it = config.begin(); it != config.end(); ++it) { + items.push_back(it->as<YamlSNIConfig::Item>()); + } + } catch (std::exception &ex) { + return ts::Errata::Message(1, 1, ex.what()); + } + + return ts::Errata(); +} + +TsEnumDescriptor LEVEL_DESCRIPTOR = {{{"NONE", 0}, {"MODERATE", 1}, {"STRICT", 2}}}; + +std::set<std::string> valid_sni_config_keys = {TS_fqdn, TS_disable_H2, TS_verify_client, + TS_tunnel_route, TS_verify_origin_server, TS_client_cert}; + +namespace YAML +{ +template <> struct convert<YamlSNIConfig::Item> { + static bool + decode(const Node &node, YamlSNIConfig::Item &item) + { + for (auto &&item : node) { + if (std::none_of(valid_sni_config_keys.begin(), valid_sni_config_keys.end(), + [&item](std::string s) { return s == item.first.as<std::string>(); })) { + throw std::runtime_error("unsupported key " + item.first.as<std::string>()); + } + } + + if (node[TS_fqdn]) { + item.fqdn = node[TS_fqdn].as<std::string>(); + } + if (node[TS_disable_H2]) { + item.fqdn = node[TS_disable_H2].as<bool>(); + } + + // enum + if (node[TS_verify_client]) { + auto value = node[TS_verify_client].as<std::string>(); + int level = LEVEL_DESCRIPTOR.get(value); + if (level < 0) { + throw std::runtime_error("unknown value " + value); + } + item.verify_client_level = static_cast<uint8_t>(level); + } + + if (node[TS_tunnel_route]) { + item.tunnel_destination = node[TS_tunnel_route].as<std::string>(); + } + + if (node[TS_verify_origin_server]) { + auto value = node[TS_verify_origin_server].as<std::string>(); + int level = LEVEL_DESCRIPTOR.get(value); + if (level < 0) { + throw std::runtime_error("unknown value " + value); + } + item.verify_origin_server = static_cast<uint8_t>(level); + } + + if (node[TS_client_cert]) { + item.client_cert = node[TS_client_cert].as<std::string>(); + } + return true; + } +}; +} // namespace YAML diff --git a/iocore/net/YamlSNIConfig.h b/iocore/net/YamlSNIConfig.h new file mode 100644 index 0000000..e3fcd11 --- /dev/null +++ b/iocore/net/YamlSNIConfig.h @@ -0,0 +1,62 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include <vector> +#include <string> + +#include "tsconfig/Errata.h" + +constexpr char TS_fqdn[] = "fqdn"; +constexpr char TS_disable_H2[] = "disable_h2"; +constexpr char TS_verify_client[] = "verify_client"; +constexpr char TS_tunnel_route[] = "tunnel_route"; +constexpr char TS_verify_origin_server[] = "verify_origin_server"; +constexpr char TS_client_cert[] = "client_cert"; + +const int start = 0; +struct YamlSNIConfig { + enum class Action { + disable_h2 = start, + verify_client, + tunnel_route, // blind tunnel action + verify_origin_server, // this applies to server side vc only + client_cert + + }; + enum class Level { NONE = 0, MODERATE, STRICT }; + + YamlSNIConfig() {} + + struct Item { + std::string fqdn; + bool disable_h2 = false; + uint8_t verify_client_level = 0; + std::string tunnel_destination; + uint8_t verify_origin_server = 0; + std::string client_cert; + }; + + ts::Errata loader(const char *cfgFilename); + + std::vector<YamlSNIConfig::Item> items; +}; diff --git a/lib/Makefile.am b/lib/Makefile.am index df3b438..5af2b38 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -26,16 +26,16 @@ if BUILD_WCCP SUBDIRS += wccp endif -#if BUILD_YAML_CPP -SUBDIRS += yamlcpp -#endif - LOCAL = if BUILD_LUAJIT LOCAL += all-luajit endif +if BUILD_YAML_CPP +LOCAL += yamlcpp +endif + # Support verbose LuaJIT builds with "make V=1". luajit__v_ = $(luajit__v_$(AM_DEFAULT_VERBOSITY)) luajit__v_0 = @ @@ -62,7 +62,9 @@ all-luajit: XCFLAGS="" \ LDFLAGS="$(LUAJIT_LDFLAGS)" \ Q=$(luajit__v_$(V)) + $(MAKE) -C yamlcpp clean-local: test "$(top_srcdir)" != "$(top_builddir)" || (cd "$(top_builddir)/$(subdir)/luajit" && $(MAKE) CC="$(CC)" clean) test "$(top_srcdir)" = "$(top_builddir)" || rm -rf "$(top_builddir)/$(subdir)/luajit" + $(MAKE) -C yamlcpp clean diff --git a/lib/ts/EnumDescriptor.h b/lib/ts/EnumDescriptor.h new file mode 100644 index 0000000..274ba92 --- /dev/null +++ b/lib/ts/EnumDescriptor.h @@ -0,0 +1,61 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include <string_view> +#include <unordered_map> + +#include "ts/HashFNV.h" + +/// Hash functor for @c string_view +inline size_t +TsLuaConfigSVHash(std::string_view const &sv) +{ + ATSHash64FNV1a h; + h.update(sv.data(), sv.size()); + return h.get(); +} + +class TsEnumDescriptor +{ +public: + struct Pair { + std::string_view key; + int value; + }; + TsEnumDescriptor(std::initializer_list<Pair> pairs) : values{pairs.size(), &TsLuaConfigSVHash}, keys{pairs.size()} + { + for (auto &p : pairs) { + values[p.key] = p.value; + keys[p.value] = p.key; + } + } + std::unordered_map<std::string_view, int, size_t (*)(std::string_view const &)> values; + std::unordered_map<int, std::string_view> keys; + int + get(std::string_view key) + { + auto it = values.find(key); + if (it != values.end()) { + return it->second; + } + return -1; + } +}; diff --git a/proxy/config/logging.config.default b/proxy/config/logging.config.default index 7f623fa..6147ae5 100644 --- a/proxy/config/logging.config.default +++ b/proxy/config/logging.config.default @@ -1,157 +1,97 @@ --- Custom log configuration. +# Custom log configuration. --- Predefined Functions. --- --- format(table) --- Returns a format object. The given table supports the following fields: --- Format (string, required): log format string --- Interval (number, optional): log aggregation interval --- --- filter.accept(string) --- Create a filter object that accepts logs matching the filter string. --- --- filter.reject(string) --- Create a filter object that drops logs matching the filter string. --- --- filter.wipe(string) --- Create a filter object that wipes (read: removes) the log fields specified in the filter string. --- --- log.ascii(table) --- Creates an ASCII logging object. --- --- log.binary(table) --- Creates a binary logging object. --- --- log.pipe(table) --- Creates a logging object that logs to a pipe. --- --- The log.* functions accept a table that supports the following fields: --- Filename (string, required): --- Format (string or format object, required): --- Header (string, optional): --- RollingEnabled (boolean, optional): --- RollingIntervalSec (number, optional): --- RollingOffsetHr (number, optional): --- RollingSizeMb (number, optional): --- Filters (array of filter objects, optional): --- CollationHosts (array of strings, optional): --- This parameter may be either a single string or an array of entries. --- Entries may be strings or arrays of strings. A string specifies a --- single collation host, which is equivalent to providing an array --- containing a single string. --- --- If multiple entries are given, multiple collation hosts are configured --- and each log entry will be forwarded to every host. --- --- If an entry is an array of strings, this defines a collation host --- failover group. The first array entry is the primary collation host --- and the remaining entries are attached as ordered failover hosts --- that will be attempted if the primary host fails. --- --- A single collation host with failover: --- { {'logs-1.example.com:4567', 'logs-2.example.com:4567'} } --- --- Multiple collation hosts: --- {'logs-1.example.com:4567', 'logs-2.example.com:4567'} --- --- Multiple collation hosts with some failover: --- {'logs-1.example.com:4567', { 'logs-2.example.com:4567', 'logs-2a.example.com:4567'} } +# The log.* functions accept a table that supports the following fields: +# Filename (string, required): +# Format (string or format object, required): +# Header (string, optional): +# RollingEnabled (boolean, optional): +# RollingIntervalSec (number, optional): +# RollingOffsetHr (number, optional): +# RollingSizeMb (number, optional): +# Filters (array of filter objects, optional): +# CollationHosts (array of strings, optional): +# This parameter may be either a single string or an array of entries. +# Entries may be strings or arrays of strings. A string specifies a +# single collation host, which is equivalent to providing an array +# containing a single string. +# +# If multiple entries are given, multiple collation hosts are configured +# and each log entry will be forwarded to every host. +# +# If an entry is an array of strings, this defines a collation host +# failover group. The first array entry is the primary collation host +# and the remaining entries are attached as ordered failover hosts +# that will be attempted if the primary host fails. +# +# A single collation host with failover: +# { {'logs-1.example.com:4567', 'logs-2.example.com:4567'} } +# +# Multiple collation hosts: +# {'logs-1.example.com:4567', 'logs-2.example.com:4567'} +# +# Multiple collation hosts with some failover: +# {'logs-1.example.com:4567', { 'logs-2.example.com:4567', 'logs-2a.example.com:4567'} } --- Predefined variables. --- --- log.roll.none (number) --- RollingEnabled value to disable all log rolling. --- --- log.roll.time (number) --- RollingEnabled value. Roll at a certain time frequency, specified --- by RollingIntervalSec, and RollingOffsetHr. --- --- log.roll.size (number) --- RollingEnabled value. Roll when the size exceeds RollingSizeMb. --- --- log.roll.both (number) --- RollingEnabled value. Roll when either the specified rolling --- time is reached or the specified file size is reached. --- --- log.roll.any (number) --- RollingEnabled value. Roll the log file when the specified --- rolling time is reached if the size of the file equals or exceeds --- the specified size. --- --- log.protocol.http (number) --- Server protocol constants for constructing %<etype> filters. +# Predefined variables. +# +# log.roll.none (number) +# RollingEnabled value to disable all log rolling. +# +# log.roll.time (number) +# RollingEnabled value. Roll at a certain time frequency, specified +# by RollingIntervalSec, and RollingOffsetHr. +# +# log.roll.size (number) +# RollingEnabled value. Roll when the size exceeds RollingSizeMb. +# +# log.roll.both (number) +# RollingEnabled value. Roll when either the specified rolling +# time is reached or the specified file size is reached. +# +# log.roll.any (number) +# RollingEnabled value. Roll the log file when the specified +# rolling time is reached if the size of the file equals or exceeds +# the specified size. +# +# log.protocol.http (number) +# Server protocol constants for constructing %<etype> filters. --- Removed parameters. --- --- The following logging object parameters that were supported in --- the XML configuration file have been removed. --- --- Protocols: --- The list of protocols to log is a comma separated list of the --- protocols that the object logs. If the log object has no --- Protocol tag, then it logs all protocols. The Protocol tag --- simply provides an easy way to create a filter that accepts --- the specified protocols. --- --- The log object Protocols parameter is no longer supported. This --- parameter can be implemented with a filter on the %<etype> (log --- entry type) log field, eg. --- --- all = filter.accept(string.format('%%<etype> CONTAIN %d', log.protocol.http)) --- http_only = filter.accept(string.format('%%<etype> CONTAIN %d', log.protocol.http)) --- --- ServerHosts: --- This parameter provides an easy way to create a filter that logs --- only the requests to hosts in the comma separated list. Only entries --- from the named servers will be included in the log file. Servers --- can only be specified by name, not by IP address. --- --- The log object ServerHosts parameter is no longer supported. It can --- implemented with a filter on the %<shn> (server host name) log --- field, eg. --- --- hosts = filter.accept('%<shn> CASE_INSENSITIVE_CONTAIN host1,host2,host3,etc") +formats: + # WebTrends Enhanced Log Format. + # + # The following is compatible with the WebTrends Enhanced Log Format. + # If you want to generate a log that can be parsed by WebTrends + # reporting tools, simply create a log that uses this format. + - name: welf + format: |- + id=firewall time="%<cqtd> %<cqtt>" fw=%<phn> pri=6 proto=%<cqus> duration=%<ttmsf> sent=%<psql> rcvd=%<cqhl> src=%<chi> dst=%<shi> dstname=%<shn> user=%<caun> op=%<cqhm> arg="%<cqup>" result=%<pssc> ref="%<{Referer}cqh>" agent="%<{user-agent}cqh>" cache=%<crc> + # Squid Log Format with seconds resolution timestamp. + # The following is the squid format but with a seconds-only timestamp + # (cqts) instead of a seconds and milliseconds timestamp (cqtq). + - name: squid_seconds_only_timestamp + format: '%<cqts> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<pqsn> %<psct>' --- WebTrends Enhanced Log Format. --- --- The following is compatible with the WebTrends Enhanced Log Format. --- If you want to generate a log that can be parsed by WebTrends --- reporting tools, simply create a log that uses this format. -welf = format { - Format = 'id=firewall time="%<cqtd> %<cqtt>" fw=%<phn> pri=6 proto=%<cqus> duration=%<ttmsf> sent=%<psql> rcvd=%<cqhl> src=%<chi> dst=%<shi> dstname=%<shn> user=%<caun> op=%<cqhm> arg="%<cqup>" result=%<pssc> ref="%<{Referer}cqh>" agent="%<{user-agent}cqh>" cache=%<crc>' -} + # Squid Log Format. + - name: squid + format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<pqsn> %<psct>' --- Squid Log Format with seconds resolution timestamp. --- --- The following is the squid format but with a seconds-only timestamp --- (cqts) instead of a seconds and milliseconds timestamp (cqtq). -squid_seconds_only_timestamp = format { - Format = '%<cqts> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<pqsn> %<psct>' -} + # Common Log Format. + - name: common + format: '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl>' --- Squid Log Format. -squid = format { - Format = '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<pqsn> %<psct>' -} + # Extended Log Format. + - name: 'extended' + formatX: |- + %<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts> --- Common Log Format. -common = format { - Format = '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl>' -} + # Extended2 Log Formats + - name: "extended2" + format: '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts> %<phr> %<cfsc> %<pfsc> %<crc>' --- Extended Log Format. -extended = format { - Format = '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts>' -} +logs: + - filename: squid + format: quid + mode: binary --- Extended2 Log Formats -extended2 = format { - Format = '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts> %<phr> %<cfsc> %<pfsc> %<crc>' -} -log.binary { - Format = squid, - Filename = 'squid' -} - --- vim: set ft=lua : +# vim: set ft=yaml : diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc index f52517b..3c10148 100644 --- a/proxy/logging/LogConfig.cc +++ b/proxy/logging/LogConfig.cc @@ -51,6 +51,8 @@ #include "LogCollationAccept.h" +#include "YamlLogConfig.h" + #define DISK_IS_CONFIG_FULL_MESSAGE \ "Access logging to local log directory suspended - " \ "configured space allocation exhausted." @@ -472,6 +474,12 @@ LogConfig::display(FILE *fd) fprintf(fd, "\n"); fprintf(fd, "************ Log Objects (%u objects) ************\n", (unsigned int)log_object_manager.get_num_objects()); log_object_manager.display(fd); + + fprintf(fd, "************ Filter List (%u filters) ************\n", filter_list.count()); + filter_list.display(fd); + + fprintf(fd, "************ Format List (%u formats) ************\n", format_list.count()); + format_list.display(fd); } //----------------------------------------------------------------------------- @@ -488,7 +496,9 @@ LogConfig::setup_log_objects() { Debug("log", "creating objects..."); - // Evaluate logging.config to construct the custome log objects. + filter_list.clear(); + + // Evaluate logging.config to construct the custom log objects. evaluate_config(); // Open local pipes so readers can see them. @@ -915,16 +925,22 @@ LogConfig::update_space_used() bool LogConfig::evaluate_config() { - BindingInstance binding; ats_scoped_str path(RecConfigReadConfigPath("proxy.config.log.config.filename", "logging.config")); - - if (!binding.construct()) { - Fatal("failed to initialize Lua runtime"); + struct stat sbuf; + if (stat(path.get(), &sbuf) == -1 && errno == ENOENT) { + Warning("logging configuration '%s' doesn't exist", path.get()); + return false; } - if (MakeLogBindings(binding, this)) { - return binding.require(path.get()); + Note("loading logging.config"); + YamlLogConfig y(this); + + bool zret = y.parse(path.get()); + if (zret) { + Note("logging.config done reloading!"); + } else { + Note("failed to reload logging.config"); } - return false; + return zret; } diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h index ead00e3..816e22b 100644 --- a/proxy/logging/LogConfig.h +++ b/proxy/logging/LogConfig.h @@ -26,6 +26,7 @@ #include "ts/ink_platform.h" #include "P_RecProcess.h" #include "ProxyConfig.h" +#include "LogObject.h" /* Instead of enumerating the stats in DynamicStats.h, each module needs to enumerate its stats separately and register them with librecords @@ -168,6 +169,9 @@ public: LogObjectManager log_object_manager; + LogFilterList filter_list; + LogFormatList format_list; + int log_buffer_size; int max_secs_per_buffer; int max_space_mb_for_logs; diff --git a/proxy/logging/LogField.cc b/proxy/logging/LogField.cc index e458f17..049f2b3 100644 --- a/proxy/logging/LogField.cc +++ b/proxy/logging/LogField.cc @@ -678,9 +678,9 @@ LogField::valid_aggregate_name(char *name) } bool -LogField::fieldlist_contains_aggregates(char *fieldlist) +LogField::fieldlist_contains_aggregates(const char *fieldlist) { - char *match; + const char *match; for (unsigned i = 1; i < countof(aggregate_names); i++) { if ((match = strstr(fieldlist, aggregate_names[i])) != nullptr) { diff --git a/proxy/logging/LogField.h b/proxy/logging/LogField.h index 20df638..c892091 100644 --- a/proxy/logging/LogField.h +++ b/proxy/logging/LogField.h @@ -177,7 +177,7 @@ public: static void init_milestone_container(void); static Container valid_container_name(char *name); static Aggregate valid_aggregate_name(char *name); - static bool fieldlist_contains_aggregates(char *fieldlist); + static bool fieldlist_contains_aggregates(const char *fieldlist); private: char *m_name; diff --git a/proxy/logging/LogFilter.cc b/proxy/logging/LogFilter.cc index f8d6bad..8c60d82 100644 --- a/proxy/logging/LogFilter.cc +++ b/proxy/logging/LogFilter.cc @@ -1045,7 +1045,7 @@ LogFilterList::toss_this_entry(LogAccess *lad) -------------------------------------------------------------------------*/ LogFilter * -LogFilterList::find_by_name(char *name) +LogFilterList::find_by_name(const char *name) { for (LogFilter *f = first(); f; f = next(f)) { if (strcmp(f->name(), name) == 0) { diff --git a/proxy/logging/LogFilter.h b/proxy/logging/LogFilter.h index 1597ab6..259f9dd 100644 --- a/proxy/logging/LogFilter.h +++ b/proxy/logging/LogFilter.h @@ -263,7 +263,7 @@ public: void add(LogFilter *filter, bool copy = true); bool toss_this_entry(LogAccess *lad); bool wipe_this_entry(LogAccess *lad); - LogFilter *find_by_name(char *name); + LogFilter *find_by_name(const char *name); void clear(); LogFilter * diff --git a/proxy/logging/Makefile.am b/proxy/logging/Makefile.am index 90a7caf..24f4aa7 100644 --- a/proxy/logging/Makefile.am +++ b/proxy/logging/Makefile.am @@ -72,7 +72,10 @@ liblogging_a_SOURCES = \ LogSock.cc \ LogSock.h \ LogUtils.cc \ - LogUtils.h + LogUtils.h \ + YamlLogConfig.cc \ + YamlLogConfigDecoders.cc \ + YamlLogConfig.h liblogcollation_a_SOURCES = \ LogCollationAccept.cc \ diff --git a/proxy/logging/YamlLogConfig.cc b/proxy/logging/YamlLogConfig.cc new file mode 100644 index 0000000..83cc384 --- /dev/null +++ b/proxy/logging/YamlLogConfig.cc @@ -0,0 +1,247 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "YamlLogConfig.h" +#include "YamlLogConfigDecoders.h" + +#include "LogConfig.h" +#include "LogObject.h" + +#include "ts/EnumDescriptor.h" + +#include <yaml-cpp/yaml.h> +#include <algorithm> +#include <memory> + +bool +YamlLogConfig::parse(const char *cfgFilename) +{ + bool result; + try { + result = loadLogConfig(cfgFilename); + } catch (std::exception &ex) { + Error("%s", ex.what()); + result = false; + } + return result; +} + +bool +YamlLogConfig::loadLogConfig(const char *cfgFilename) +{ + YAML::Node config = YAML::LoadFile(cfgFilename); + + if (config.IsNull()) { + Warning("logging.config is empty"); + return false; + } + + if (!config.IsMap()) { + Error("malformed logging.config file; expected a map"); + return false; + } + + auto formats = config["formats"]; + for (auto it = formats.begin(); it != formats.end(); ++it) { + auto fmt = it->as<std::unique_ptr<LogFormat>>().release(); + if (fmt->valid()) { + cfg->format_list.add(fmt, false); + + if (is_debug_tag_set("log")) { + printf("The following format was added to the global format list\n"); + fmt->display(stdout); + } + } else { + Note("Format named \"%s\" will not be active; not a valid format", fmt->name() ? fmt->name() : ""); + delete fmt; + } + } + + auto filters = config["filters"]; + for (auto it = filters.begin(); it != filters.end(); ++it) { + auto filter = it->as<std::unique_ptr<LogFilter>>().release(); + + if (filter) { + cfg->filter_list.add(filter, false); + + if (is_debug_tag_set("xml")) { + printf("The following filter was added to the global filter list\n"); + filter->display(stdout); + } + } + } + + auto logs = config["logs"]; + for (auto it = logs.begin(); it != logs.end(); ++it) { + auto obj = decodeLogObject(*it); + if (obj) { + cfg->log_object_manager.manage_object(obj); + } + } + return true; +} + +TsEnumDescriptor ROLLING_MODE = {{{"none", 0}, {"time", 1}, {"size", 2}, {"both", 3}, {"any", 4}}}; + +std::set<std::string> valid_log_object_keys = { + "filename", "format", "mode", "header", "rolling_enabled", "rolling_interval_sec", + "rolling_offset_hr", "rolling_size_mb", "filters", "collation_hosts"}; + +LogObject * +YamlLogConfig::decodeLogObject(const YAML::Node &node) +{ + for (auto &&item : node) { + if (std::none_of(valid_log_object_keys.begin(), valid_log_object_keys.end(), + [&item](std::string s) { return s == item.first.as<std::string>(); })) { + throw std::runtime_error("log: unsupported key '" + item.first.as<std::string>() + "'"); + } + } + + if (!node["format"]) { + throw std::runtime_error("missing 'format' argument"); + } + std::string format = node["format"].as<std::string>(); + + if (!node["filename"]) { + throw std::runtime_error("missing 'filename' argument"); + } + + std::string header; + if (node["header"]) { + header = node["header"].as<std::string>(); + } + + std::string filename = node["filename"].as<std::string>(); + LogFormat *fmt = cfg->format_list.find_by_name(format.c_str()); + if (!fmt) { + Error("Format %s is not a known format; cannot create LogObject", format.c_str()); + return nullptr; + } + + // file format + LogFileFormat file_type = LOG_FILE_ASCII; // default value + if (node["mode"]) { + const char *mode_str = node["mode"].as<std::string>().c_str(); + file_type = (strncasecmp(mode_str, "bin", 3) == 0 || (mode_str[0] == 'b' && mode_str[1] == 0) ? + LOG_FILE_BINARY : + (strcasecmp(mode_str, "ascii_pipe") == 0 ? LOG_FILE_PIPE : LOG_FILE_ASCII)); + } + + int obj_rolling_enabled = 0; + int obj_rolling_interval_sec = cfg->rolling_interval_sec; + int obj_rolling_offset_hr = cfg->rolling_offset_hr; + int obj_rolling_size_mb = cfg->rolling_size_mb; + + if (node["rolling_enabled"]) { + auto value = node["rolling_enabled"].as<std::string>(); + obj_rolling_enabled = ROLLING_MODE.get(value); + if (obj_rolling_enabled < 0) { + throw std::runtime_error("unknown value " + value); + } + } + if (node["rolling_interval_sec"]) { + obj_rolling_interval_sec = node["rolling_interval_sec"].as<int>(); + } + if (node["rolling_offset_hr"]) { + obj_rolling_offset_hr = node["rolling_offset_hr"].as<int>(); + } + if (node["rolling_size_mb"]) { + obj_rolling_size_mb = node["rolling_size_mb"].as<int>(); + } + + if (!LogRollingEnabledIsValid(obj_rolling_enabled)) { + Warning("Invalid log rolling value '%d' in log object", obj_rolling_enabled); + } + + auto logObject = new LogObject(fmt, Log::config->logfile_dir, filename.c_str(), file_type, header.c_str(), + (Log::RollingEnabledValues)obj_rolling_enabled, Log::config->collation_preproc_threads, + obj_rolling_interval_sec, obj_rolling_offset_hr, obj_rolling_size_mb); + + // filters + auto filters = node["filters"]; + if (!filters) { + return logObject; + } + + if (!filters.IsSequence()) { + throw std::runtime_error("'filters' should be a list"); + } + + for (auto &&filter : filters) { + const char *filter_name = filter.as<std::string>().c_str(); + LogFilter *f = cfg->filter_list.find_by_name(filter_name); + if (!f) { + Warning("Filter %s is not a known filter; cannot add to this LogObject", filter_name); + } else { + logObject->add_filter(f); + } + } + + auto collation_host_list = node["collation_hosts"]; + if (!collation_host_list) { + return logObject; + } + + if (!collation_host_list.IsSequence()) { + throw std::runtime_error("'collation_hosts' should be a list of collation_host objects"); + } + + for (auto &&collation_host : collation_host_list) { + if (!collation_host["host"]) { + Warning("no collation 'host' name; cannot add this Collation host"); + continue; + } + + auto collation_host_name = collation_host["host"].as<std::string>(); + + LogHost *lh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); + if (!lh->set_name_or_ipstr(collation_host_name.c_str())) { + Warning("Could not set \"%s\" as collation host", collation_host_name.c_str()); + delete lh; + continue; + } + + logObject->add_loghost(lh, false); + if (!collation_host["failover"]) { + continue; + } + + if (!collation_host["failover"].IsSequence()) { + delete lh; + throw std::runtime_error("'failover' should be a list of host names"); + } + + LogHost *prev = lh; + for (auto &&failover_host : collation_host["failover"]) { + auto failover_host_name = failover_host.as<std::string>(); + LogHost *flh = new LogHost(logObject->get_full_filename(), logObject->get_signature()); + if (!flh->set_name_or_ipstr(failover_host_name.c_str())) { + Warning("Could not set \"%s\" as a failover host", failover_host_name.c_str()); + delete flh; + continue; + } + prev->failover_link.next = flh; + prev = flh; + } + } + + return logObject; +} diff --git a/proxy/logging/YamlLogConfig.h b/proxy/logging/YamlLogConfig.h new file mode 100644 index 0000000..323b406 --- /dev/null +++ b/proxy/logging/YamlLogConfig.h @@ -0,0 +1,46 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +class LogConfig; +class LogObject; +namespace YAML +{ +class Node; +}; + +class YamlLogConfig +{ +public: + YamlLogConfig(LogConfig *logConfig) : cfg(logConfig){}; + bool parse(const char *cfgFilename); + +private: + bool loadLogConfig(const char *cfgFilename); + + LogObject *decodeLogObject(const YAML::Node &node); + + YamlLogConfig(const YamlLogConfig &) = delete; + YamlLogConfig &operator=(const YamlLogConfig &) = delete; + + LogConfig *cfg; +}; diff --git a/proxy/logging/YamlLogConfigDecoders.cc b/proxy/logging/YamlLogConfigDecoders.cc new file mode 100644 index 0000000..d1853c1 --- /dev/null +++ b/proxy/logging/YamlLogConfigDecoders.cc @@ -0,0 +1,118 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "YamlLogConfigDecoders.h" + +#include "LogConfig.h" +#include "LogObject.h" + +#include <yaml-cpp/yaml.h> +#include <algorithm> +#include <memory> + +std::set<std::string> valid_log_format_keys = {"name", "format", "interval"}; +std::set<std::string> valid_log_filter_keys = {"name", "action", "condition"}; + +namespace YAML +{ +bool +convert<std::unique_ptr<LogFormat>>::decode(const Node &node, std::unique_ptr<LogFormat> &logFormat) +{ + for (auto &&item : node) { + if (std::none_of(valid_log_format_keys.begin(), valid_log_format_keys.end(), + [&item](std::string s) { return s == item.first.as<std::string>(); })) { + throw std::runtime_error("format: unsupported key '" + item.first.as<std::string>() + "'"); + } + } + + if (!node["format"]) { + throw std::runtime_error("missing 'format' argument"); + } + std::string format = node["format"].as<std::string>(); + + std::string name; + if (node["name"]) { + name = node["name"].as<std::string>(); + } + + // if the format_str contains any of the aggregate operators, + // we need to ensure that an interval was specified. + if (LogField::fieldlist_contains_aggregates(format.c_str())) { + if (!node["interval"]) { + Note("'interval' attribute missing for LogFormat object" + " %s that contains aggregate operators: %s", + name.c_str(), format.c_str()); + return false; + } + } + + unsigned interval = 0; + if (node["interval"]) { + interval = node["interval"].as<unsigned>(); + } + + logFormat.reset(new LogFormat(name.c_str(), format.c_str(), interval)); + + return true; +} + +bool +convert<std::unique_ptr<LogFilter>>::decode(const Node &node, std::unique_ptr<LogFilter> &logFilter) +{ + for (auto &&item : node) { + if (std::none_of(valid_log_filter_keys.begin(), valid_log_filter_keys.end(), + [&item](std::string s) { return s == item.first.as<std::string>(); })) { + throw std::runtime_error("filter: unsupported key '" + item.first.as<std::string>() + "'"); + } + } + + // we require all keys for LogFilter + for (auto &&item : valid_log_filter_keys) { + if (!node[item]) { + throw std::runtime_error("missing '" + item + "' argument"); + } + } + + auto name = node["name"].as<std::string>(); + auto action = node["action"].as<std::string>(); + auto condition = node["condition"].as<std::string>(); + + auto action_str = action.c_str(); + LogFilter::Action act = LogFilter::REJECT; /* lv: make gcc happy */ + int i; + for (i = 0; i < LogFilter::N_ACTIONS; i++) { + if (strcasecmp(action_str, LogFilter::ACTION_NAME[i]) == 0) { + act = (LogFilter::Action)i; + break; + } + } + + if (i == LogFilter::N_ACTIONS) { + Warning("%s is not a valid filter action value; cannot create filter %s.", action_str, name.c_str()); + return false; + } + + logFilter.reset(LogFilter::parse(name.c_str(), act, condition.c_str())); + + return true; +} + +} // namespace YAML diff --git a/proxy/logging/YamlLogConfigDecoders.h b/proxy/logging/YamlLogConfigDecoders.h new file mode 100644 index 0000000..108ba17 --- /dev/null +++ b/proxy/logging/YamlLogConfigDecoders.h @@ -0,0 +1,41 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include <memory> + +#include <yaml-cpp/yaml.h> + +class LogFormat; +class LogFilter; + +namespace YAML +{ +template <> struct convert<std::unique_ptr<LogFormat>> { + static bool decode(const Node &node, std::unique_ptr<LogFormat> &logFormat); +}; + +template <> struct convert<std::unique_ptr<LogFilter>> { + static bool decode(const Node &node, std::unique_ptr<LogFilter> &logFilter); +}; + +} // namespace YAML diff --git a/src/traffic_logcat/Makefile.inc b/src/traffic_logcat/Makefile.inc index f46c098..bc91386 100644 --- a/src/traffic_logcat/Makefile.inc +++ b/src/traffic_logcat/Makefile.inc @@ -50,7 +50,8 @@ traffic_logcat_traffic_logcat_LDADD += \ endif traffic_logcat_traffic_logcat_LDADD += \ - @LIBTCL@ @HWLOC_LIBS@\ + @LIBTCL@ @HWLOC_LIBS@ \ + @LIB_YAMLCPP@ \ @LIBPROFILER@ -lm if SYSTEM_LUAJIT diff --git a/src/traffic_logstats/Makefile.inc b/src/traffic_logstats/Makefile.inc index 347c005..b69bb64 100644 --- a/src/traffic_logstats/Makefile.inc +++ b/src/traffic_logstats/Makefile.inc @@ -55,6 +55,7 @@ endif traffic_logstats_traffic_logstats_LDADD += \ @LIBTCL@ @HWLOC_LIBS@ \ + @LIB_YAMLCPP@ \ @LIBPROFILER@ -lm if SYSTEM_LUAJIT diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 19f8aec..28d825b 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -95,6 +95,7 @@ traffic_server_traffic_server_LDADD += \ @LIBLZMA@ \ @LIBPROFILER@ \ @OPENSSL_LIBS@ \ + @LIB_YAMLCPP@ \ -lm if BUILD_LUAJIT diff --git a/tests/gold_tests/h2/httpbin.test.py b/tests/gold_tests/h2/httpbin.test.py index f615507..9738739 100644 --- a/tests/gold_tests/h2/httpbin.test.py +++ b/tests/gold_tests/h2/httpbin.test.py @@ -67,14 +67,15 @@ ts.Disk.records_config.update({ }) ts.Disk.logging_config.AddLines( ''' --- Extended Log Format. -access = format { - Format = '[%<cqtn>] %<cqtx> %<cqpv> %<cqssv> %<cqssc> %<crc> %<pssc> %<pscl>' -} +formats: + # Extended Log Format. + - name: access + format: |- +[%<cqtn>] %<cqtx> %<cqpv> %<cqssv> %<cqssc> %<crc> %<pssc> %<pscl> -log.ascii { - Format = access, - Filename = 'access' +logs: + - filename: access + format: access } '''.split("\n") ) diff --git a/tests/gold_tests/logging/ccid_ctid.test.py b/tests/gold_tests/logging/ccid_ctid.test.py index b990891..762529c 100644 --- a/tests/gold_tests/logging/ccid_ctid.test.py +++ b/tests/gold_tests/logging/ccid_ctid.test.py @@ -59,14 +59,14 @@ ts.Disk.ssl_multicert_config.AddLine( ) ts.Disk.logging_config.AddLines( - '''custom = format { - Format = "%<ccid> %<ctid>" -} - -log.ascii { - Format = custom, - Filename = 'test_ccid_ctid' -}'''.split("\n") + ''' +formats: + - name: custom + format: "%<ccid> %<ctid>" +logs: + - filename: test_ccid_ctid + format: custom +'''.split("\n") ) tr = Test.AddTestRun() diff --git a/tests/gold_tests/logging/custom-log.test.py b/tests/gold_tests/logging/custom-log.test.py index 683a251..e297348 100644 --- a/tests/gold_tests/logging/custom-log.test.py +++ b/tests/gold_tests/logging/custom-log.test.py @@ -37,14 +37,14 @@ ts.Disk.remap_config.AddLine( ) ts.Disk.logging_config.AddLines( - '''custom = format { - Format = "%<hii> %<hiih>" -} - -log.ascii { - Format = custom, - Filename = 'test_log_field' -}'''.split("\n") + ''' +formats: + - name: custom + format: "%<hii> %<hiih>" +logs: + - filename: test_log_field + format: custom +'''.split("\n") ) # ######################################################################### diff --git a/tests/gold_tests/logging/log-field.test.py b/tests/gold_tests/logging/log-field.test.py index 66cd7c7..81359f0 100644 --- a/tests/gold_tests/logging/log-field.test.py +++ b/tests/gold_tests/logging/log-field.test.py @@ -57,14 +57,13 @@ ts.Disk.remap_config.AddLine( ts.Disk.logging_config.AddLines( ''' -custom = format { - Format = '%<{Content-Type}essh>' -} +formats: + - name: custom + format: '%<{Content-Type}essh>' -log.ascii { - Format = custom, - Filename = 'field-test' -} +logs: + - filename: field-test + format: custom '''.split("\n") ) -- To stop receiving notification emails like this one, please contact [email protected].
