This is an automated email from the ASF dual-hosted git repository.
cmcfarlen pushed a commit to branch 10.0.x
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit 5c82874dc17df4add57090123495809a7ad9c647
Author: Leif Hedstrom <zw...@apache.org>
AuthorDate: Thu Apr 11 10:07:13 2024 -0600
Allow Cripts to be used directly as @plugin (#11192)
* Allow for traffic_server to compile Cripts directly
This relies on an external script / application to do the
actual work. If not configured, no new behavior is introduced.
* Fixes from review
(cherry picked from commit defd0f2f5b8efcf366a4651f30a4ba9d266b1b99)
---
doc/admin-guide/files/records.yaml.en.rst | 14 +++++
include/proxy/http/remap/PluginDso.h | 2 +-
include/proxy/http/remap/PluginFactory.h | 2 +
include/proxy/http/remap/RemapPluginInfo.h | 2 +-
src/proxy/http/remap/PluginDso.cc | 24 ++++++--
src/proxy/http/remap/PluginFactory.cc | 24 +++++++-
src/proxy/http/remap/RemapPluginInfo.cc | 4 +-
src/proxy/http/remap/UrlRewrite.cc | 19 ++++++
src/records/RecordsConfig.cc | 2 +
src/traffic_server/traffic_server.cc | 2 +-
tools/cripts/compiler.sh | 97 ++++++++++++++++++++++++++++++
11 files changed, 182 insertions(+), 10 deletions(-)
diff --git a/doc/admin-guide/files/records.yaml.en.rst
b/doc/admin-guide/files/records.yaml.en.rst
index a75647aba4..6b2172bb1b 100644
--- a/doc/admin-guide/files/records.yaml.en.rst
+++ b/doc/admin-guide/files/records.yaml.en.rst
@@ -4895,6 +4895,20 @@ Plug-in Configuration
Enables (``1``) or disables (``0``) the dynamic reload feature for remap
plugins (`remap.config`). Global plugins (`plugin.config`) do not have
dynamic reload feature yet.
+.. ts:cv:: CONFIG proxy.config.plugin.compiler_path STRING ""
+
+ Specifies an optional compiler tool path for compiling plugins. This tool
should
+ be an executable, which takes two arguments:
+
+ === ======================================================================
+ Arg Description
+ === ======================================================================
+ 1 This is the path to the source file, which should be compiled
+ 2 This is the path to the DSO file, which will be created and loaded
+ === ======================================================================
+
+ The script should exit with a status code of ``0`` if the compilation was
successful.
+
.. ts:cv:: CONFIG proxy.config.plugin.vc.default_buffer_index INT 8
:reloadable:
:overridable:
diff --git a/include/proxy/http/remap/PluginDso.h
b/include/proxy/http/remap/PluginDso.h
index 41f18b943d..9c0d6e452d 100644
--- a/include/proxy/http/remap/PluginDso.h
+++ b/include/proxy/http/remap/PluginDso.h
@@ -66,7 +66,7 @@ public:
virtual ~PluginDso();
/* DSO Load, unload, get symbols from DSO */
- virtual bool load(std::string &error);
+ virtual bool load(std::string &error, const fs::path &compilerPath);
virtual bool unload(std::string &error);
bool isLoaded();
bool getSymbol(const char *symbol, void *&address, std::string &error) const;
diff --git a/include/proxy/http/remap/PluginFactory.h
b/include/proxy/http/remap/PluginFactory.h
index 5a63a13f41..1ebcadd81e 100644
--- a/include/proxy/http/remap/PluginFactory.h
+++ b/include/proxy/http/remap/PluginFactory.h
@@ -95,6 +95,7 @@ public:
virtual ~PluginFactory();
PluginFactory &setRuntimeDir(const fs::path &runtimeDir);
+ PluginFactory &setCompilerPath(const fs::path &compilerPath);
PluginFactory &addSearchDir(const fs::path &searchDir);
RemapPluginInst *getRemapPlugin(const fs::path &configPath, int argc, char
**argv, std::string &error, bool dynamicReloadEnabled);
@@ -112,6 +113,7 @@ protected:
std::vector<fs::path> _searchDirs; /** @brief ordered list of search paths
where we look for plugins */
fs::path _runtimeDir; /** @brief the path where we would create
a temporary copies of the plugins to load */
+ fs::path _compilerPath; /** @brief the compilation script to use
for cripts and other non-DSO plugins */
PluginInstList _instList;
diff --git a/include/proxy/http/remap/RemapPluginInfo.h
b/include/proxy/http/remap/RemapPluginInfo.h
index dda7695ce6..73a91643f0 100644
--- a/include/proxy/http/remap/RemapPluginInfo.h
+++ b/include/proxy/http/remap/RemapPluginInfo.h
@@ -83,7 +83,7 @@ public:
~RemapPluginInfo();
/* Overload to add / execute remap plugin specific tasks during the plugin
loading */
- bool load(std::string &error) override;
+ bool load(std::string &error, const fs::path &compilerPath) override;
/* Used by the factory to invoke callbacks during plugin load, init and
unload */
bool init(std::string &error) override;
diff --git a/src/proxy/http/remap/PluginDso.cc
b/src/proxy/http/remap/PluginDso.cc
index 9da092a790..4f59167164 100644
--- a/src/proxy/http/remap/PluginDso.cc
+++ b/src/proxy/http/remap/PluginDso.cc
@@ -38,6 +38,8 @@
#define PluginError Error
#endif
+#include <cstdlib>
+
namespace
{
@@ -69,7 +71,7 @@ PluginDso::~PluginDso()
}
bool
-PluginDso::load(std::string &error)
+PluginDso::load(std::string &error, const fs::path &compilerPath)
{
/* Clear all errors */
error.clear();
@@ -89,13 +91,27 @@ PluginDso::load(std::string &error)
} else {
PluginDbg(_dbg_ctl(), "plugin '%s' effective path: %s",
_configPath.c_str(), _effectivePath.c_str());
- /* Copy the installed plugin DSO to a runtime directory if dynamic reload
enabled */
std::error_code ec;
- if (isDynamicReloadEnabled() && !copy(_effectivePath, _runtimePath, ec)) {
+
+ if (!_effectivePath.string().ends_with(".so")) {
+ if (!isDynamicReloadEnabled()) {
+ concat_error(error, "Dynamic reload must be enabled for Cript files");
+ result = false;
+ } else {
+ std::string command = compilerPath.string() + " " +
_effectivePath.string() + " " + _runtimePath.string();
+
+ if (std::system(command.c_str()) != 0) {
+ concat_error(error, "Compile script failed");
+ result = false;
+ }
+ }
+ } else if (isDynamicReloadEnabled() && !copy(_effectivePath, _runtimePath,
ec)) {
concat_error(error, "failed to create a copy");
concat_error(error, ec.message());
result = false;
- } else {
+ }
+
+ if (result) {
PluginDbg(_dbg_ctl(), "plugin '%s' runtime path: %s",
_configPath.c_str(), _runtimePath.c_str());
/* Save the time for later checking if DSO got modified in consecutive
DSO reloads */
diff --git a/src/proxy/http/remap/PluginFactory.cc
b/src/proxy/http/remap/PluginFactory.cc
index 399da3c042..74b25a8673 100644
--- a/src/proxy/http/remap/PluginFactory.cc
+++ b/src/proxy/http/remap/PluginFactory.cc
@@ -124,6 +124,14 @@ PluginFactory::setRuntimeDir(const fs::path &runtimeDir)
return *this;
}
+PluginFactory &
+PluginFactory::setCompilerPath(const fs::path &compilerPath)
+{
+ _compilerPath = compilerPath;
+ PluginDbg(_dbg_ctl(), "set plugin compiler path %s", compilerPath.c_str());
+ return *this;
+}
+
const char *
PluginFactory::getUuid()
{
@@ -175,6 +183,20 @@ PluginFactory::getRemapPlugin(const fs::path &configPath,
int argc, char **argv,
runtimePath /= _runtimeDir;
runtimePath /= effectivePath.relative_path();
+ // Special case for Cripts
+ if (!runtimePath.string().ends_with(".so")) {
+ if (_compilerPath.empty()) {
+ error.assign("compiler path not set for compiling plugins");
+ return nullptr;
+ }
+
+ // Add .so to the source file, so e.g. example.cc.so. ToDo: libswoc
doesn't allow appending to the extension.
+ std::string newPath = runtimePath.string();
+
+ newPath.append(".so");
+ runtimePath = newPath;
+ }
+
fs::path parent = runtimePath.parent_path();
PluginDbg(_dbg_ctl(), "Using effectivePath: [%s] runtimePath: [%s]
parent: [%s]", effectivePath.c_str(), runtimePath.c_str(),
parent.c_str());
@@ -189,7 +211,7 @@ PluginFactory::getRemapPlugin(const fs::path &configPath,
int argc, char **argv,
plugin = new RemapPluginInfo(configPath, effectivePath, runtimePath);
if (nullptr != plugin) {
- if (plugin->load(error)) {
+ if (plugin->load(error, _compilerPath)) {
if (plugin->init(error)) {
PluginDso::loadedPlugins()->add(plugin);
inst = RemapPluginInst::init(plugin, argc, argv, error);
diff --git a/src/proxy/http/remap/RemapPluginInfo.cc
b/src/proxy/http/remap/RemapPluginInfo.cc
index 001fad8a86..849214ba47 100644
--- a/src/proxy/http/remap/RemapPluginInfo.cc
+++ b/src/proxy/http/remap/RemapPluginInfo.cc
@@ -77,11 +77,11 @@ RemapPluginInfo::RemapPluginInfo(const fs::path
&configPath, const fs::path &eff
}
bool
-RemapPluginInfo::load(std::string &error)
+RemapPluginInfo::load(std::string &error, const fs::path &compilerPath)
{
error.clear();
- if (!PluginDso::load(error)) {
+ if (!PluginDso::load(error, compilerPath)) {
return false;
}
diff --git a/src/proxy/http/remap/UrlRewrite.cc
b/src/proxy/http/remap/UrlRewrite.cc
index 80b1460749..fa13988035 100644
--- a/src/proxy/http/remap/UrlRewrite.cc
+++ b/src/proxy/http/remap/UrlRewrite.cc
@@ -87,6 +87,25 @@ UrlRewrite::load()
/* Initialize the plugin factory */
pluginFactory.setRuntimeDir(RecConfigReadRuntimeDir()).addSearchDir(RecConfigReadPluginDir());
+ // This is "optional", and this configuration is not set by default.
+ char buf[PATH_NAME_MAX];
+
+ buf[0] = '\0';
+ RecGetRecordString("proxy.config.plugin.compiler_path", buf, PATH_NAME_MAX);
+ if (strlen(buf) > 0) {
+ std::error_code ec;
+ fs::path compilerPath = fs::path(buf);
+ fs::file_status status = fs::status(compilerPath, ec);
+
+ if (ec || !swoc::file::is_regular_file(status)) {
+ Error("Configured plugin compiler path '%s' is not a regular file", buf);
+ return false;
+ } else {
+ // This also adds the configuration directory (etc/trafficserver) to
find Cripts etc.
+
pluginFactory.setCompilerPath(compilerPath).addSearchDir(RecConfigReadConfigDir());
+ }
+ }
+
/* Initialize the next hop strategy factory */
std::string sf =
RecConfigReadConfigPath("proxy.config.url_remap.strategies.filename",
"strategies.yaml");
Dbg(dbg_ctl_url_rewrite_regex, "strategyFactory file: %s", sf.c_str());
diff --git a/src/records/RecordsConfig.cc b/src/records/RecordsConfig.cc
index 77751488ee..00ac0c4de1 100644
--- a/src/records/RecordsConfig.cc
+++ b/src/records/RecordsConfig.cc
@@ -53,6 +53,8 @@ static const RecordElement RecordsConfig[] =
,
{RECT_CONFIG, "proxy.config.bin_path", RECD_STRING, TS_BUILD_BINDIR,
RECU_NULL, RR_REQUIRED, RECC_NULL, nullptr, RECA_READ_ONLY}
,
+ {RECT_CONFIG, "proxy.config.plugin.compiler_path", RECD_STRING, "",
RECU_NULL, RR_REQUIRED, RECC_NULL, nullptr, RECA_READ_ONLY}
+ ,
// Jira TS-21
{RECT_CONFIG, "proxy.config.local_state_dir", RECD_STRING,
TS_BUILD_RUNTIMEDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr,
RECA_READ_ONLY}
,
diff --git a/src/traffic_server/traffic_server.cc
b/src/traffic_server/traffic_server.cc
index f6df676b3e..5d7dd9c579 100644
--- a/src/traffic_server/traffic_server.cc
+++ b/src/traffic_server/traffic_server.cc
@@ -1036,7 +1036,7 @@ try_loading_plugin(plugin_type_t plugin_type, const
fs::path &plugin_path, std::
const auto runtime_path = temporary_directory / plugin_path.filename();
const fs::path unused_config;
auto plugin_info = std::make_unique<RemapPluginInfo>(unused_config,
plugin_path, runtime_path);
- bool loaded = plugin_info->load(error);
+ bool loaded = plugin_info->load(error, unused_config); // ToDo: Will
this ever need support for cripts
if (!fs::remove(temporary_directory, ec)) {
fprintf(stderr, "ERROR: could not remove temporary directory '%s':
%s\n", temporary_directory.c_str(), ec.message().c_str());
}
diff --git a/tools/cripts/compiler.sh b/tools/cripts/compiler.sh
new file mode 100755
index 0000000000..74e6d3dd64
--- /dev/null
+++ b/tools/cripts/compiler.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##############################################################################
+# This is an example compiler script for compiling Cripts and other plugins
+# on the fly. You would set proxy.config.plugin.compiler_path to point to a
+# customized version of this script, and it will be called whenever a plugin
+# needs to be compiled via remap.config.
+##############################################################################
+
+# Configurable parts
+ATS_ROOT=/opt/ats
+CXX=clang++
+CXXFLAGS="-std=c++20 -I/opt/homebrew/include -undefined dynamic_lookup"
+
+# Probably don't need to change these ?
+STDFLAGS="-shared -fPIC -Wall -Werror -I${ATS_ROOT}/include -L${ATS_ROOT}/lib
-lcripts"
+
+# This is optional, but if set, the script will cache the compiled shared
objects for faster restarts/reloads
+CACHE_DIR=/tmp/ats-cache
+
+# Extract the arguments, and do some sanity checks
+SOURCE=$1
+DEST=$2
+
+SOURCE_DIR=$(dirname $SOURCE)
+DEST_DIR=$(dirname $DEST)
+SOURCE_FILE=$(basename $SOURCE)
+DEST_FILE=$(basename $DEST)
+
+cd "$SOURCE_DIR"
+if [ $(pwd) != "$SOURCE_DIR" ]; then
+ echo "Failed to cd to $SOURCE_DIR"
+ exit 1
+fi
+
+cd "$DEST_DIR"
+if [ $(pwd) != "$DEST_DIR" ]; then
+ echo "Failed to cd to $DEST_DIR"
+ exit 1
+fi
+
+if [ ! -f "$SOURCE" ]; then
+ echo "Source file $SOURCE does not exist"
+ exit 1
+fi
+
+if [ "${DEST_FILE}" != "${SOURCE_FILE}.so" ]; then
+ echo "Destination file name must match source file name with .so extension"
+ exit 1
+fi
+
+cache_file=""
+if [ -d "$CACHE_DIR" ]; then
+ cache_tree="${CACHE_DIR}${SOURCE_DIR}"
+ cache_file="${cache_tree}/${DEST_FILE}"
+
+ [ -d "$cache_tree" ] || mkdir -p "$cache_tree"
+
+ if [ "$SOURCE" -ot "$cache_file" ]; then
+ cp "$cache_file" "$DEST"
+ exit 0
+ fi
+
+ DEST="$cache_file"
+fi
+
+# Compile the plugin
+${CXX} ${CXXFLAGS} ${STDFLAGS} -o $DEST $SOURCE
+exit_code=$?
+
+if [ $exit_code -ne 0 ]; then
+ echo "Compilation failed with exit code $exit_code"
+ exit $exit_code
+fi
+
+# In case we compiled to the cache, copy from the cache to the runtime
destination
+if [ -f "$cache_file" ]; then
+ cp "$cache_file" "${DEST_DIR}/${DEST_FILE}"
+fi
+
+exit 0
