This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 1e7d1b3d1 [webserver] handlers for application/octet-stream content
1e7d1b3d1 is described below
commit 1e7d1b3d117ca137bdce3d2ec549416d242bdd02
Author: Alexey Serbin <[email protected]>
AuthorDate: Wed Jun 7 20:38:10 2023 -0700
[webserver] handlers for application/octet-stream content
This patch introduces application/octet-stream (a.k.a. binary data)
handlers for the embedded web server.
As a particular application of the newly introduced feature, this patch
also adds /ipki-ca-cert-der endpoint to the master's embedded webserver.
It outputs the same IPKI CA certificate as the /ipki-ca-cert and
/ipki-ca-cert-pem endpoints, but in DER format. This will be used
in a changelist where Java client needs to import Kudu cluster's CA
certificate into its trust chain when using JWT for authentication.
As it turns out, the 'standard' security providers for Java runtime
don't provide enough functionality to conveniently work with X509 in PEM
format (yes, there is BouncyCastle, but I'm not sure we want to
introduce such a dependency just for the convenience of working with
certificates in PEM format at a few call sites in the Java client).
To cover the newly introduced functionality, an extra test has been added
into security-itest and PeriodicWebUIChecker has been updated.
Change-Id: I894d8e00943617cb80ec5aa14a15db3448ad9252
Reviewed-on: http://gerrit.cloudera.org:8080/20023
Tested-by: Kudu Jenkins
Reviewed-by: Zoltan Chovan <[email protected]>
Reviewed-by: Attila Bukor <[email protected]>
---
src/kudu/integration-tests/security-itest.cc | 58 ++++++++++++++++++++++++++++
src/kudu/master/master_path_handlers.cc | 32 +++++++++++----
src/kudu/master/master_path_handlers.h | 8 ++--
src/kudu/mini-cluster/webui_checker.h | 2 +
src/kudu/server/webserver.cc | 47 ++++++++++++++++++----
src/kudu/server/webserver.h | 23 +++++++----
src/kudu/util/web_callback_registry.h | 12 +++++-
7 files changed, 154 insertions(+), 28 deletions(-)
diff --git a/src/kudu/integration-tests/security-itest.cc
b/src/kudu/integration-tests/security-itest.cc
index a560b0707..fdf54619f 100644
--- a/src/kudu/integration-tests/security-itest.cc
+++ b/src/kudu/integration-tests/security-itest.cc
@@ -1122,6 +1122,64 @@ TEST_F(SecurityITest, IPKICACert) {
});
}
+// Test the endpoints of the embedded master's webserver that output the IPKI
+// CA certificates in PEM and DER formats, /ipki-ca-cert[-pem] and
+// ipki-ca-cert-der, correspondingly. Make sure the end-points are functional
+// and their output is consistent.
+TEST_F(SecurityITest, IPKICACertWebServerEndPoints) {
+ // Need just a single master.
+ cluster_opts_.num_masters = 1;
+ // No need to involve tablet servers in this scenario.
+ cluster_opts_.num_tablet_servers = 0;
+
+ ASSERT_OK(StartCluster());
+
+ const auto fetch = [c = cluster_.get()](const string& path, string* out) {
+ const auto& http_hp = c->master()->bound_http_hostport();
+ string url = Substitute("http://$0/$1";, http_hp.ToString(), path);
+ EasyCurl curl;
+ faststring dst;
+ auto res = curl.FetchURL(url, &dst);
+ *out = dst.ToString();
+ return res;
+ };
+
+ ASSERT_OK(cluster_->master()->WaitForCatalogManager());
+
+ security::Cert ca_cert_0;
+ {
+ string ca_cert_str_pem;
+ ASSERT_OK(fetch("ipki-ca-cert", &ca_cert_str_pem));
+ ASSERT_OK(ca_cert_0.FromString(ca_cert_str_pem,
security::DataFormat::PEM));
+ }
+
+ security::Cert ca_cert_1;
+ {
+ string ca_cert_str_pem;
+ ASSERT_OK(fetch("ipki-ca-cert-pem", &ca_cert_str_pem));
+ ASSERT_OK(ca_cert_1.FromString(ca_cert_str_pem,
security::DataFormat::PEM));
+ }
+
+ security::Cert ca_cert_2;
+ {
+ string ca_cert_str_der;
+ ASSERT_OK(fetch("ipki-ca-cert-der", &ca_cert_str_der));
+ ASSERT_OK(ca_cert_2.FromString(ca_cert_str_der,
security::DataFormat::DER));
+ }
+
+ // Using (security::Cert --> string) conversion to compare canonical string
+ // representations of the captured CA certificates.
+ string ca_cert_str_0;
+ ASSERT_OK(ca_cert_0.ToString(&ca_cert_str_0, security::DataFormat::PEM));
+ string ca_cert_str_1;
+ ASSERT_OK(ca_cert_1.ToString(&ca_cert_str_1, security::DataFormat::PEM));
+ string ca_cert_str_2;
+ ASSERT_OK(ca_cert_2.ToString(&ca_cert_str_2, security::DataFormat::PEM));
+
+ ASSERT_EQ(ca_cert_str_0, ca_cert_str_1);
+ ASSERT_EQ(ca_cert_str_1, ca_cert_str_2);
+}
+
class EncryptionPolicyTest :
public SecurityITest,
public ::testing::WithParamInterface<tuple<
diff --git a/src/kudu/master/master_path_handlers.cc
b/src/kudu/master/master_path_handlers.cc
index 31f78a689..3bc817aef 100644
--- a/src/kudu/master/master_path_handlers.cc
+++ b/src/kudu/master/master_path_handlers.cc
@@ -77,10 +77,9 @@
#include "kudu/util/url-coding.h"
#include "kudu/util/web_callback_registry.h"
-namespace kudu {
-
-using consensus::ConsensusStatePB;
-using consensus::RaftPeerPB;
+using kudu::consensus::ConsensusStatePB;
+using kudu::consensus::RaftPeerPB;
+using kudu::security::DataFormat;
using std::array;
using std::map;
using std::ostringstream;
@@ -90,6 +89,7 @@ using std::string;
using std::vector;
using strings::Substitute;
+namespace kudu {
namespace master {
namespace {
@@ -606,6 +606,7 @@ void MasterPathHandlers::HandleMasters(const
Webserver::WebRequest& /*req*/,
}
void MasterPathHandlers::HandleIpkiCaCert(
+ DataFormat cert_format,
const Webserver::WebRequest& /*req*/,
Webserver::PrerenderedWebResponse* resp) {
ostringstream& out = resp->output;
@@ -622,14 +623,18 @@ void MasterPathHandlers::HandleIpkiCaCert(
}
const auto& cert = ca->ca_cert();
string cert_str;
- if (auto s = cert.ToString(&cert_str, security::DataFormat::PEM); !s.ok()) {
- auto err = s.CloneAndPrepend("could not convert CA cert to PEM format");
+ if (auto s = cert.ToString(&cert_str, cert_format); !s.ok()) {
+ auto err = s.CloneAndPrepend(
+ Substitute("could not convert CA cert to $0 format",
+ security::DataFormatToString(cert_format)));
LOG(ERROR) << err.ToString();
resp->status_code = HttpStatusCode::InternalServerError;
out << "ERROR: " << err.ToString();
return;
}
- RemoveExtraWhitespace(&cert_str);
+ if (cert_format == DataFormat::PEM) {
+ RemoveExtraWhitespace(&cert_str);
+ }
out << cert_str;
}
@@ -850,9 +855,20 @@ Status MasterPathHandlers::Register(Webserver* server) {
server->RegisterPrerenderedPathHandler(
"/ipki-ca-cert", "IPKI CA certificate",
[this](const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
- this->HandleIpkiCaCert(req, resp);
+ this->HandleIpkiCaCert(DataFormat::PEM, req, resp);
},
false /*is_styled*/, true /*is_on_nav_bar*/);
+ server->RegisterPrerenderedPathHandler(
+ "/ipki-ca-cert-pem", "IPKI CA certificate (PEM format)",
+ [this](const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
+ this->HandleIpkiCaCert(DataFormat::PEM, req, resp);
+ },
+ false /*is_styled*/, false /*is_on_nav_bar*/);
+ server->RegisterBinaryDataPathHandler(
+ "/ipki-ca-cert-der", "IPKI CA certificate (DER format)",
+ [this](const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
+ this->HandleIpkiCaCert(DataFormat::DER, req, resp);
+ });
server->RegisterPrerenderedPathHandler(
"/dump-entities", "Dump Entities",
[this](const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
diff --git a/src/kudu/master/master_path_handlers.h
b/src/kudu/master/master_path_handlers.h
index feced1417..8c89d3429 100644
--- a/src/kudu/master/master_path_handlers.h
+++ b/src/kudu/master/master_path_handlers.h
@@ -14,14 +14,14 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
-#ifndef KUDU_MASTER_MASTER_PATH_HANDLERS_H
-#define KUDU_MASTER_MASTER_PATH_HANDLERS_H
+#pragma once
#include <string>
#include <utility>
#include "kudu/gutil/macros.h"
#include "kudu/server/webserver.h"
+#include "kudu/util/openssl_util.h"
#include "kudu/util/status.h"
namespace kudu {
@@ -52,7 +52,8 @@ class MasterPathHandlers {
Webserver::WebResponse* resp);
void HandleMasters(const Webserver::WebRequest& req,
Webserver::WebResponse* resp);
- void HandleIpkiCaCert(const Webserver::WebRequest& req,
+ void HandleIpkiCaCert(security::DataFormat cert_format,
+ const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp);
void HandleDumpEntities(const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp);
@@ -81,4 +82,3 @@ class MasterPathHandlers {
} // namespace master
} // namespace kudu
-#endif /* KUDU_MASTER_MASTER_PATH_HANDLERS_H */
diff --git a/src/kudu/mini-cluster/webui_checker.h
b/src/kudu/mini-cluster/webui_checker.h
index 9f368a216..88c64ca5b 100644
--- a/src/kudu/mini-cluster/webui_checker.h
+++ b/src/kudu/mini-cluster/webui_checker.h
@@ -37,6 +37,8 @@ class PeriodicWebUIChecker {
const std::vector<std::string>& master_pages = {
"/dump-entities",
"/ipki-ca-cert",
+ "/ipki-ca-cert-der",
+ "/ipki-ca-cert-pem",
"/masters",
"/mem-trackers",
"/metrics",
diff --git a/src/kudu/server/webserver.cc b/src/kudu/server/webserver.cc
index 2eb3e91c8..d94b1db1d 100644
--- a/src/kudu/server/webserver.cc
+++ b/src/kudu/server/webserver.cc
@@ -661,7 +661,7 @@ void Webserver::SendResponse(struct sq_connection*
connection,
ostringstream oss;
Status s = zlib::CompressLevel(uncompressed, 1, &oss);
- if (s.ok()) {
+ if (PREDICT_TRUE(s.ok())) {
resp->output.str(oss.str());
is_compressed = true;
} else {
@@ -686,10 +686,29 @@ void Webserver::SendResponse(struct sq_connection*
connection,
// Write the headers to the buffer first, then write the body.
oss << Substitute("HTTP/1.1 $0\r\n",
HttpStatusCodeToString(resp->status_code));
- oss << Substitute("Content-Type: $0\r\n",
- mode == StyleMode::STYLED ? "text/html" : "text/plain");
+
+ // The "Content-Type" is defined by the value of the 'mode' parameter.
+ const char* content_type = nullptr;
+ switch (mode) {
+ case StyleMode::STYLED:
+ content_type = "text/html";
+ break;
+ case StyleMode::UNSTYLED:
+ content_type = "text/plain";
+ break;
+ case StyleMode::BINARY:
+ content_type = "application/octet-stream";
+ break;
+ default:
+ LOG(FATAL) << "unexpected style mode: " << static_cast<uint32_t>(mode);
+ break; // unreachable
+ }
+ oss << Substitute("Content-Type: $0\r\n", content_type);
+
oss << Substitute("Content-Length: $0\r\n", body.length());
- if (is_compressed) oss << "Content-Encoding: gzip\r\n";
+ if (is_compressed) {
+ oss << "Content-Encoding: gzip\r\n";
+ }
if (PREDICT_TRUE(FLAGS_webserver_enable_csp)) {
// TODO(aserbin): add information on when to update the SHA hash and
// how to do so (ideally, the exact command line)
@@ -699,14 +718,15 @@ void Webserver::SendResponse(struct sq_connection*
connection,
}
oss << Substitute("X-Frame-Options: $0\r\n",
FLAGS_webserver_x_frame_options);
static const unordered_set<string> kInvalidHeaders = {
+ "Content-Encoding",
"Content-Length",
"Content-Type",
"Content-Security-Policy",
- "X-Frame-Options"
+ "X-Frame-Options",
};
for (const auto& entry : resp->response_headers) {
// It's forbidden to override the above headers.
- if (ContainsKey(kInvalidHeaders, entry.first)) {
+ if (PREDICT_FALSE(ContainsKey(kInvalidHeaders, entry.first))) {
LOG(FATAL) << Substitute("Reserved header $0 was overridden by handler",
entry.first);
}
@@ -731,6 +751,10 @@ void Webserver::AddKnoxVariables(const WebRequest& req,
EasyJson* json) {
}
}
+string Webserver::MustachePartialTag(const string& path) {
+ return Substitute("{{> $0.mustache}}", path);
+}
+
void Webserver::RegisterPathHandler(const string& path, const string& alias,
const PathHandlerCallback& callback, bool is_styled, bool is_on_nav_bar) {
string render_path = (path == "/") ? "/home" : path;
@@ -757,8 +781,15 @@ void Webserver::RegisterPrerenderedPathHandler(const
string& path, const string&
InsertOrDie(&path_handlers_, path, new PathHandler(is_styled, is_on_nav_bar,
alias, callback));
}
-string Webserver::MustachePartialTag(const string& path) const {
- return Substitute("{{> $0.mustache}}", path);
+void Webserver::RegisterBinaryDataPathHandler(
+ const string& path,
+ const string& alias,
+ const PrerenderedPathHandlerCallback& callback) {
+ std::lock_guard<RWMutex> l(lock_);
+ InsertOrDie(&path_handlers_, path, new PathHandler(false /*is_styled*/,
+ false /*is_on_nav_bar*/,
+ alias,
+ callback));
}
bool Webserver::MustacheTemplateAvailable(const string& path) const {
diff --git a/src/kudu/server/webserver.h b/src/kudu/server/webserver.h
index a8fd8404a..073ee8d84 100644
--- a/src/kudu/server/webserver.h
+++ b/src/kudu/server/webserver.h
@@ -80,6 +80,12 @@ class Webserver : public WebCallbackRegistry {
bool is_styled,
bool is_on_nav_bar) override;
+ // Register route 'path' for application/octet-stream (binary data)
responses.
+ void RegisterBinaryDataPathHandler(
+ const std::string& path,
+ const std::string& alias,
+ const PrerenderedPathHandlerCallback& callback) override;
+
// Change the footer HTML to be displayed at the bottom of all styled web
pages.
void set_footer_html(const std::string& html);
@@ -113,13 +119,13 @@ class Webserver : public WebCallbackRegistry {
private:
// If true, the page appears is rendered styled.
- bool is_styled_;
+ const bool is_styled_;
// If true, the page appears in the navigation bar.
- bool is_on_nav_bar_;
+ const bool is_on_nav_bar_;
// Alias used when displaying this link on the nav bar.
- std::string alias_;
+ const std::string alias_;
// Callback to render output for this page.
PrerenderedPathHandlerCallback callback_;
@@ -128,15 +134,15 @@ class Webserver : public WebCallbackRegistry {
// Add any necessary Knox-related variables to 'json' based on the headers
in 'args'.
static void AddKnoxVariables(const WebRequest& req, EasyJson* json);
+ // Returns a mustache tag that renders the partial at path when
+ // passed to mustache::RenderTemplate.
+ static std::string MustachePartialTag(const std::string& path);
+
bool static_pages_available() const;
// Build the string to pass to mongoose specifying where to bind.
Status BuildListenSpec(std::string* spec) const WARN_UNUSED_RESULT;
- // Returns a mustache tag that renders the partial at path when
- // passed to mustache::RenderTemplate.
- std::string MustachePartialTag(const std::string& path) const;
-
// Returns whether or not a mustache template corresponding
// to the given path can be found.
bool MustacheTemplateAvailable(const std::string& path) const;
@@ -184,9 +190,12 @@ class Webserver : public WebCallbackRegistry {
// parsed the request yet (e.g. an early error out).
//
// If 'mode' is STYLED, includes page styling elements like CSS, navigation
bar, etc.
+ //
+ // BINARY is a rare case when a binary data is sent as a response.
enum class StyleMode {
STYLED,
UNSTYLED,
+ BINARY,
};
void SendResponse(struct sq_connection* connection,
PrerenderedWebResponse* resp,
diff --git a/src/kudu/util/web_callback_registry.h
b/src/kudu/util/web_callback_registry.h
index 39a80626f..7b3e49bea 100644
--- a/src/kudu/util/web_callback_registry.h
+++ b/src/kudu/util/web_callback_registry.h
@@ -86,7 +86,8 @@ class WebCallbackRegistry {
// Additional headers added to the HTTP response.
ArgumentMap response_headers;
- // The fully-rendered HTML response body.
+ // The fully-rendered HTML response body or a binary blob in case of
+ // responses with 'application/octet-stream' Content-Type.
std::ostringstream output;
};
@@ -124,6 +125,15 @@ class WebCallbackRegistry {
bool is_styled,
bool is_on_nav_bar) = 0;
+ // Register a callback for a URL path that returns binary data, a.k.a. octet
+ // stream. Such a path is not supposed to be exposed on the navigation bar
+ // of the Web UI, and the data is sent as-is with the HTTP response with no
+ // rendering assumed.
+ virtual void RegisterBinaryDataPathHandler(
+ const std::string& path,
+ const std::string& alias,
+ const PrerenderedPathHandlerCallback& callback) = 0;
+
// Returns true if 'req' was proxied via Knox, false otherwise.
static bool IsProxiedViaKnox(const WebRequest& req);
};
