This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/arrow-flight-sql-postgresql.git
The following commit(s) were added to refs/heads/main by this push:
new c603719 Add support for TLS (#46)
c603719 is described below
commit c603719442b904a95b432a221d1f6cf90e22dfb1
Author: Sutou Kouhei <[email protected]>
AuthorDate: Mon Aug 14 15:32:26 2023 +0900
Add support for TLS (#46)
Closes GH-38
---
.editorconfig | 10 +++-
dev/prepare-tls.sh | 87 ++++++++++++++++++++++++++++++++++
.editorconfig => dev/run-postgresql.sh | 38 ++++++++++++---
src/afs.cc | 32 +++++++++++++
test/helper/sandbox.rb | 55 +++++++++++++++++----
test/test-flight-sql.rb | 2 +-
6 files changed, 207 insertions(+), 17 deletions(-)
diff --git a/.editorconfig b/.editorconfig
index 362c371..4075833 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -19,8 +19,16 @@ root = true
[*.{c,cc,h}]
-indent_style = tab
+end_of_line = lf
indent_size = 4
+indent_style = tab
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.sh]
+
end_of_line = lf
+indent_size = 2
+indent_style = space
insert_final_newline = true
trim_trailing_whitespace = true
diff --git a/dev/prepare-tls.sh b/dev/prepare-tls.sh
new file mode 100755
index 0000000..c781ffb
--- /dev/null
+++ b/dev/prepare-tls.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+if [ $# -ne 3 ]; then
+ echo "Usage: $0 CA_NAME SERVER_NAME CLIENT_NAME"
+ echo " e.g.: $0 root.example.com server.example.com client.example.com"
+ exit 1
+fi
+
+root_name=$1
+server_name=$2
+client_name=$3
+
+openssl req \
+ -new \
+ -nodes \
+ -text \
+ -out root.csr \
+ -keyout root.key \
+ -subj "/CN=${root_name}"
+chmod go-rwx root.key
+
+openssl x509 \
+ -req \
+ -in root.csr \
+ -text \
+ -days 3650 \
+ -extfile /etc/ssl/openssl.cnf \
+ -extensions v3_ca \
+ -signkey root.key \
+ -out root.crt
+
+openssl req \
+ -new \
+ -nodes \
+ -text \
+ -out server.csr \
+ -keyout server.key \
+ -subj "/CN=${server_name}"
+chmod og-rwx server.key
+
+openssl x509 \
+ -req \
+ -in server.csr \
+ -text \
+ -days 365 \
+ -CA root.crt \
+ -CAkey root.key \
+ -CAcreateserial \
+ -out server.crt
+
+openssl req \
+ -new \
+ -nodes \
+ -text \
+ -out client.csr \
+ -keyout client.key \
+ -subj "/CN=${client_name}"
+chmod og-rwx client.key
+
+openssl x509 \
+ -req \
+ -in client.csr \
+ -text \
+ -days 365 \
+ -CA root.crt \
+ -CAkey root.key \
+ -CAcreateserial \
+ -out client.crt
diff --git a/.editorconfig b/dev/run-postgresql.sh
old mode 100644
new mode 100755
similarity index 52%
copy from .editorconfig
copy to dev/run-postgresql.sh
index 362c371..209bfb9
--- a/.editorconfig
+++ b/dev/run-postgresql.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,12 +17,34 @@
# specific language governing permissions and limitations
# under the License.
-root = true
+set -eu
+
+if [ $# -ne 4 ]; then
+ echo "Usage: $0 DATA_DIRECTORY CA_NAME SERVER_NAME CLIENT_NAME"
+ echo " e.g.: $0 /tmp/afs root.example.com server.example.com
client.example.com"
+ exit 1
+fi
+
+data_directory=$1
+root_name=$2
+server_name=$3
+client_name=$4
+
+base_directory="$(cd "$(dirname "$0")" && pwd)"
-[*.{c,cc,h}]
+rm -rf "${data_directory}"
-indent_style = tab
-indent_size = 4
-end_of_line = lf
-insert_final_newline = true
-trim_trailing_whitespace = true
+initdb \
+ --locale=C \
+ --set=arrow_flight_sql.uri=grpc+tls://${server_name}:15432 \
+ --set=shared_preload_libraries=arrow_flight_sql \
+ --set=ssl=on \
+ --set=ssl_ca_file=root.crt \
+ "${data_directory}"
+pushd "${data_directory}"
+"${base_directory}/prepare-tls.sh" \
+ "${root_name}" \
+ "${server_name}" \
+ "${client_name}"
+popd
+LANG=C postgres -D "${data_directory}"
diff --git a/src/afs.cc b/src/afs.cc
index 603eba0..d267a43 100644
--- a/src/afs.cc
+++ b/src/afs.cc
@@ -25,8 +25,10 @@ extern "C"
#include <lib/dshash.h>
#include <libpq/crypt.h>
#include <libpq/libpq-be.h>
+#include <libpq/libpq.h>
#include <miscadmin.h>
#include <postmaster/bgworker.h>
+#include <postmaster/postmaster.h>
#include <storage/ipc.h>
#include <storage/latch.h>
#include <storage/lwlock.h>
@@ -54,6 +56,8 @@ extern "C"
#include <cinttypes>
#include <condition_variable>
+#include <fstream>
+#include <iterator>
#include <random>
#include <sstream>
@@ -1654,12 +1658,40 @@ afs_server_internal(Proxy* proxy)
{
ARROW_ASSIGN_OR_RAISE(auto location,
arrow::flight::Location::Parse(URI));
arrow::flight::FlightServerOptions options(location);
+ if (EnableSSL)
+ {
+ arrow::flight::CertKeyPair certificate;
+ if (ssl_cert_file)
+ {
+ std::ifstream input(ssl_cert_file);
+ if (input)
+ {
+ certificate.pem_cert =
+
std::string(std::istreambuf_iterator<char>{input}, {});
+ }
+ }
+ if (ssl_key_file)
+ {
+ std::ifstream input(ssl_key_file);
+ if (input)
+ {
+ certificate.pem_key =
+
std::string(std::istreambuf_iterator<char>{input}, {});
+ }
+ }
+ if (!certificate.pem_cert.empty() &&
!certificate.pem_key.empty())
+ {
+
options.tls_certificates.push_back(std::move(certificate));
+ }
+ }
options.auth_handler =
std::make_unique<arrow::flight::NoOpAuthHandler>();
options.middleware.push_back(
{"header-auth",
std::make_shared<HeaderAuthServerMiddlewareFactory>(proxy)});
FlightSQLServer flightSQLServer(proxy);
ARROW_RETURN_NOT_OK(flightSQLServer.Init(options));
+ ereport(LOG, (errmsg("listening on %s for Apache Arrow Flight SQL",
URI)));
+
while (!GotSIGTERM)
{
WaitLatch(MyLatch, WL_LATCH_SET | WL_EXIT_ON_PM_DEATH, -1,
PG_WAIT_EXTENSION);
diff --git a/test/helper/sandbox.rb b/test/helper/sandbox.rb
index 6aec966..a320683 100644
--- a/test/helper/sandbox.rb
+++ b/test/helper/sandbox.rb
@@ -96,7 +96,7 @@ module Helper
include CommandRunnable
attr_reader :dir
- attr_reader :host
+ attr_reader :address
attr_reader :port
attr_reader :flight_sql_port
attr_reader :flight_sql_uri
@@ -107,7 +107,7 @@ module Helper
@dir = nil
@log_base_name = "postgresql.log"
@log_path = nil
- @host = "127.0.0.1"
+ @address = "127.0.0.1"
@port = nil
@flight_sql_port = nil
@flight_sql_uri = nil
@@ -129,10 +129,14 @@ module Helper
socket_dir = File.join(@dir, "socket")
@port = port
@pgpass = Tempfile.new("arrow-flight-sql-test-pgpass")
- @pgpass.puts("#{@host}:#{@port}:*:#{@user}:#{@password}")
+ @pgpass.puts("#{@address}:#{@port}:*:#{@user}:#{@password}")
@pgpass.close
@flight_sql_port = flight_sql_port
- @flight_sql_uri = "grpc://#{@host}:#{@flight_sql_port}"
+ if use_tls?
+ @flight_sql_uri = "grpc+tls://#{@address}:#{@flight_sql_port}"
+ else
+ @flight_sql_uri = "grpc://#{@address}:#{@flight_sql_port}"
+ end
Tempfile.create("arrow-flight-sql-test-password") do |password|
password.print(@password)
password.close
@@ -143,19 +147,24 @@ module Helper
"--pwfile", password.path,
"-D", @dir)
end
+ prepare_tls if use_tls?
FileUtils.mkdir_p(socket_dir)
postgresql_conf = File.join(@dir, "postgresql.conf")
File.open(postgresql_conf, "a") do |conf|
- conf.puts("listen_addresses = '#{@host}'")
+ conf.puts("listen_addresses = '#{@address}'")
conf.puts("port = #{@port}")
unless windows?
conf.puts("unix_socket_directories = '#{socket_dir}'")
end
+ if use_tls?
+ conf.puts("ssl = on")
+ conf.puts("ssl_ca_file = 'root.crt'")
+ end
conf.puts("logging_collector = on")
conf.puts("log_filename = '#{@log_base_name}'")
conf.puts("shared_preload_libraries = " +
"'#{shared_preload_libraries.join(",")}'")
- conf.puts("arrow_flight_sql.uri = #{@flight_sql_uri}")
+ conf.puts("arrow_flight_sql.uri = '#{@flight_sql_uri}'")
yield(conf) if block_given?
end
pg_hba_conf = File.join(@dir, "pg_hba.conf")
@@ -197,7 +206,7 @@ module Helper
"PGPASSFILE" => @pgpass.path,
},
"psql",
- "--host", @host,
+ "--host", @address,
"--port", @port.to_s,
"--username", @user,
"--dbname", db,
@@ -209,7 +218,12 @@ module Helper
end
def flight_client
- @flight_client ||= ArrowFlight::Client.new(@flight_sql_uri)
+ @flight_client ||=
+ ArrowFlight::Client.new(@flight_sql_uri, flight_client_options)
+ end
+
+ def flight_client_options
+ @flight_client_options ||= create_flight_client_options
end
def flight_sql_client
@@ -225,6 +239,31 @@ module Helper
def windows?
/mingw|mswin|cygwin/.match?(RUBY_PLATFORM)
end
+
+ def use_tls?
+ return false if windows?
+ ArrowFlight::ClientOptions.method_defined?(:tls_root_certificates=)
+ end
+
+ def create_flight_client_options
+ options = ArrowFlight::ClientOptions.new
+ if use_tls?
+ options.tls_root_certificates = File.read(File.join(@dir, "root.crt"))
+ options.override_host_name = "server.example.com"
+ end
+ options
+ end
+
+ def prepare_tls
+ prepare_tls_sh = File.join(__dir__, "..", "..", "dev", "prepare-tls.sh")
+ prepare_tls_sh = File.expand_path(prepare_tls_sh)
+ Dir.chdir(@dir) do
+ run_command(prepare_tls_sh,
+ "root.example.com",
+ "server.example.com",
+ "client.example.com")
+ end
+ end
end
module Sandbox
diff --git a/test/test-flight-sql.rb b/test/test-flight-sql.rb
index 6997460..4237dd4 100644
--- a/test/test-flight-sql.rb
+++ b/test/test-flight-sql.rb
@@ -53,7 +53,7 @@ class FlightSQLTest < Test::Unit::TestCase
end
def test_isnert_int32
- unless filght_sql_client.respond_to?(:execute_update)
+ unless flight_sql_client.respond_to?(:execute_update)
omit("red-arrow-flight-sql 13.0.0 or later is required")
end
