This is an automated email from the ASF dual-hosted git repository.
curth pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 152d25cbd feat(csharp/src/Drivers/Apache): Custom ssl server
certificate validation for Spark, Impala & Hive (#2610)
152d25cbd is described below
commit 152d25cbdc67730998534284c0b111b988a0e0e7
Author: Sudhir Reddy Emmadi <[email protected]>
AuthorDate: Wed Mar 26 03:18:52 2025 +0530
feat(csharp/src/Drivers/Apache): Custom ssl server certificate validation
for Spark, Impala & Hive (#2610)
Co-authored-by: Sudhir Emmadi <[email protected]>
---
.../Drivers/Apache/Hive2/HiveServer2Connection.cs | 7 +-
.../Apache/Hive2/HiveServer2HttpConnection.cs | 26 +-----
.../Drivers/Apache/Hive2/HiveServer2Parameters.cs | 10 +-
.../src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs | 101 +++++++++++++++++++++
.../Drivers/Apache/Hive2/HiveServer2TlsOption.cs | 53 -----------
csharp/src/Drivers/Apache/Hive2/README.md | 6 +-
.../Drivers/Apache/Impala/ImpalaHttpConnection.cs | 27 +-----
.../Apache/Impala/ImpalaStandardConnection.cs | 1 -
csharp/src/Drivers/Apache/Spark/README.md | 6 +-
.../Drivers/Apache/Spark/SparkHttpConnection.cs | 27 +-----
csharp/src/Drivers/Apache/Spark/SparkParameters.cs | 1 -
.../Apache/Hive2/HiveServer2ParametersTest.cs | 31 -------
.../Apache/Hive2/HiveServer2TestEnvironment.cs | 4 -
.../Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs | 59 ++++++++++++
.../Drivers/Apache/Spark/SparkTestEnvironment.cs | 4 -
15 files changed, 191 insertions(+), 172 deletions(-)
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
b/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
index e182adb4f..ab3efea31 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2Connection.cs
@@ -342,7 +342,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
protected internal DataTypeConversion DataTypeConversion { get; set; }
= DataTypeConversion.None;
- protected internal HiveServer2TlsOption TlsOptions { get; set; } =
HiveServer2TlsOption.Empty;
+ protected internal TlsProperties TlsOptions { get; set; } = new
TlsProperties();
protected internal int ConnectTimeoutMilliseconds { get; set; } =
ConnectTimeoutMillisecondsDefault;
@@ -731,7 +731,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
return fileVersionInfo.ProductVersion ??
GetProductVersionDefault();
}
- protected static Uri GetBaseAddress(string? uri, string? hostName,
string? path, string? port, string hostOptionName)
+ protected static Uri GetBaseAddress(string? uri, string? hostName,
string? path, string? port, string hostOptionName, bool isTlsEnabled)
{
// Uri property takes precedent.
if (!string.IsNullOrWhiteSpace(uri))
@@ -755,8 +755,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
bool isPortSet = !string.IsNullOrEmpty(port);
bool isValidPortNumber = int.TryParse(port, out int portNumber) &&
portNumber > 0;
- bool isDefaultHttpsPort = !isPortSet || (isValidPortNumber &&
portNumber == 443);
- string uriScheme = isDefaultHttpsPort ? Uri.UriSchemeHttps :
Uri.UriSchemeHttp;
+ string uriScheme = isTlsEnabled ? Uri.UriSchemeHttps :
Uri.UriSchemeHttp;
int uriPort;
if (!isPortSet)
uriPort = -1;
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
b/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
index 19f799322..187e5712c 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2HttpConnection.cs
@@ -121,7 +121,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
Properties.TryGetValue(HiveServer2Parameters.Path, out string?
path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port,
HiveServer2Parameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port,
HiveServer2Parameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -129,8 +129,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
{
Properties.TryGetValue(HiveServer2Parameters.DataTypeConv, out
string? dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(HiveServer2Parameters.TLSOptions, out
string? tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(HiveServer2Parameters.ConnectTimeoutMilliseconds, out
string? connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -138,6 +136,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
? connectTimeoutMsValue
: throw new
ArgumentOutOfRangeException(HiveServer2Parameters.ConnectTimeoutMilliseconds,
connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 ..
{int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
public override AdbcStatement CreateStatement()
@@ -166,10 +165,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
HiveServer2Parameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
HiveServer2Parameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue =
GetAuthenticationHeaderValue(authTypeValue, username, password);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler =
HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
httpClientHandler.AutomaticDecompression =
DecompressionMethods.GZip | DecompressionMethods.Deflate;
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
@@ -190,23 +189,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback =
(request, certificate, chain, policyErrors) =>
- {
- if
(policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) &&
!TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned)) return false;
- if
(policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) &&
!TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch)) return false;
-
- return true;
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue?
GetAuthenticationHeaderValue(HiveServer2AuthType authType, string? username,
string? password)
{
if (!string.IsNullOrEmpty(username) &&
!string.IsNullOrEmpty(password) && (authType == HiveServer2AuthType.Empty ||
authType == HiveServer2AuthType.Basic))
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
b/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
index b9d7f2c2a..f1bb90f11 100644
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2Parameters.cs
@@ -25,7 +25,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
public const string AuthType = "adbc.hive.auth_type";
public const string TransportType = "adbc.hive.transport_type";
public const string DataTypeConv = "adbc.hive.data_type_conv";
- public const string TLSOptions = "adbc.hive.tls_options";
public const string ConnectTimeoutMilliseconds =
"adbc.hive.connect_timeout_ms";
}
@@ -47,9 +46,12 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
public const string Scalar = "scalar";
}
- public static class TlsOptions
+ public static class HttpTlsOptions
{
- public const string AllowSelfSigned = "allow_self_signed";
- public const string AllowHostnameMismatch = "allow_hostname_mismatch";
+ public const string IsTlsEnabled = "adbc.http_options.tls.enabled";
+ public const string AllowSelfSigned =
"adbc.http_options.tls.allow_self_signed";
+ public const string AllowHostnameMismatch =
"adbc.http_options.tls.allow_hostname_mismatch";
+ public const string TrustedCertificatePath =
"adbc.http_options.tls.trusted_certificate_path";
+ public const string DisableServerCertificateValidation =
"adbc.http_options.tls.disable_server_certificate_validation";
}
}
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs
b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs
new file mode 100644
index 000000000..b7e7c8b38
--- /dev/null
+++ b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsImpl.cs
@@ -0,0 +1,101 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
+using System.Net.Http;
+
+namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
+{
+ class TlsProperties
+ {
+ public bool IsTlsEnabled { get; set; }
+ public bool DisableServerCertificateValidation { get; set; }
+ public bool AllowHostnameMismatch { get; set; }
+ public bool AllowSelfSigned { get; set; }
+ public string? TrustedCertificatePath { get; set; }
+ }
+
+ static class HiveServer2TlsImpl
+ {
+ static internal TlsProperties
GetHttpTlsOptions(IReadOnlyDictionary<string, string> properties)
+ {
+ TlsProperties tlsProperties = new TlsProperties();
+ if (properties.TryGetValue(AdbcOptions.Uri, out string? uri) &&
!string.IsNullOrWhiteSpace(uri))
+ {
+ var uriValue = new Uri(uri);
+ tlsProperties.IsTlsEnabled = uriValue.Scheme ==
Uri.UriSchemeHttps || (properties.TryGetValue(HttpTlsOptions.IsTlsEnabled, out
string? isSslEnabled) && bool.TryParse(isSslEnabled, out bool isSslEnabledBool)
&& isSslEnabledBool);
+ }
+ else
+ {
+ tlsProperties.IsTlsEnabled =
properties.TryGetValue(HttpTlsOptions.IsTlsEnabled, out string? isSslEnabled)
&& bool.TryParse(isSslEnabled, out bool isSslEnabledBool) && isSslEnabledBool;
+ }
+ if (!tlsProperties.IsTlsEnabled)
+ {
+ return tlsProperties;
+ }
+ tlsProperties.IsTlsEnabled = true;
+ if
(properties.TryGetValue(HttpTlsOptions.DisableServerCertificateValidation, out
string? disableServerCertificateValidation) &&
bool.TryParse(disableServerCertificateValidation, out bool
disableServerCertificateValidationBool) &&
disableServerCertificateValidationBool)
+ {
+ tlsProperties.DisableServerCertificateValidation = true;
+ return tlsProperties;
+ }
+ tlsProperties.DisableServerCertificateValidation = false;
+ tlsProperties.AllowHostnameMismatch =
properties.TryGetValue(HttpTlsOptions.AllowHostnameMismatch, out string?
allowHostnameMismatch) && bool.TryParse(allowHostnameMismatch, out bool
allowHostnameMismatchBool) && allowHostnameMismatchBool;
+ tlsProperties.AllowSelfSigned =
properties.TryGetValue(HttpTlsOptions.AllowSelfSigned, out string?
allowSelfSigned) && bool.TryParse(allowSelfSigned, out bool
allowSelfSignedBool) && allowSelfSignedBool;
+ if (tlsProperties.AllowSelfSigned)
+ {
+ if
(!properties.TryGetValue(HttpTlsOptions.TrustedCertificatePath, out string?
trustedCertificatePath)) return tlsProperties;
+ tlsProperties.TrustedCertificatePath = trustedCertificatePath
!= "" && File.Exists(trustedCertificatePath) ? trustedCertificatePath : throw
new FileNotFoundException("Trusted certificate path is invalid or file does not
exist.");
+ }
+ return tlsProperties;
+ }
+
+ static internal HttpClientHandler NewHttpClientHandler(TlsProperties
tlsProperties)
+ {
+ HttpClientHandler httpClientHandler = new();
+ if (tlsProperties.IsTlsEnabled)
+ {
+ httpClientHandler.ServerCertificateCustomValidationCallback =
(request, certificate, chain, policyErrors) =>
+ {
+ if (policyErrors == SslPolicyErrors.None ||
tlsProperties.DisableServerCertificateValidation) return true;
+ if
(string.IsNullOrEmpty(tlsProperties.TrustedCertificatePath))
+ {
+ return
+
(!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) ||
tlsProperties.AllowSelfSigned)
+ &&
(!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) ||
tlsProperties.AllowHostnameMismatch);
+ }
+ if (certificate == null)
+ return false;
+ X509Certificate2 customCertificate = new
X509Certificate2(tlsProperties.TrustedCertificatePath);
+ X509Chain chain2 = new X509Chain();
+ chain2.ChainPolicy.ExtraStore.Add(customCertificate);
+
+ // "tell the X509Chain class that I do trust this root
certs and it should check just the certs in the chain and nothing else"
+ chain2.ChainPolicy.VerificationFlags =
X509VerificationFlags.AllowUnknownCertificateAuthority;
+
+ // Build the chain and verify
+ return chain2.Build(certificate);
+ };
+ }
+ return httpClientHandler;
+ }
+ }
+}
diff --git a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs
b/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs
deleted file mode 100644
index 84f56a485..000000000
--- a/csharp/src/Drivers/Apache/Hive2/HiveServer2TlsOption.cs
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-
-namespace Apache.Arrow.Adbc.Drivers.Apache.Hive2
-{
- [Flags]
- internal enum HiveServer2TlsOption
- {
- Empty = 0,
- AllowSelfSigned = 1,
- AllowHostnameMismatch = 2,
- }
-
- internal static class TlsOptionsParser
- {
- internal const string SupportedList = TlsOptions.AllowSelfSigned + ","
+ TlsOptions.AllowHostnameMismatch;
-
- internal static HiveServer2TlsOption Parse(string? tlsOptions)
- {
- HiveServer2TlsOption options = HiveServer2TlsOption.Empty;
- if (tlsOptions == null) return options;
-
- string[] valueList = tlsOptions.Split(',');
- foreach (string tlsOption in valueList)
- {
- options |= (tlsOption?.Trim().ToLowerInvariant()) switch
- {
- null or "" => HiveServer2TlsOption.Empty,
- TlsOptions.AllowSelfSigned =>
HiveServer2TlsOption.AllowSelfSigned,
- TlsOptions.AllowHostnameMismatch =>
HiveServer2TlsOption.AllowHostnameMismatch,
- _ => throw new
ArgumentOutOfRangeException(nameof(tlsOptions), tlsOption, "Invalid or
unsupported TLS option"),
- };
- }
- return options;
- }
- }
-}
diff --git a/csharp/src/Drivers/Apache/Hive2/README.md
b/csharp/src/Drivers/Apache/Hive2/README.md
index e9c863190..5e665d3f8 100644
--- a/csharp/src/Drivers/Apache/Hive2/README.md
+++ b/csharp/src/Drivers/Apache/Hive2/README.md
@@ -35,11 +35,15 @@ but can also be passed in the call to
`AdbcDatabase.Connect`.
| `username` | The user name used for basic authentication | |
| `password` | The password for the user name used for basic
authentication. | |
| `adbc.hive.data_type_conv` | Comma-separated list of data conversion
options. Each option indicates the type of conversion to perform on data
returned from the Hive server. <br><br>Allowed values: `none`, `scalar`.
<br><br>Option `none` indicates there is no conversion from Hive type to native
type (i.e., no conversion from String to Timestamp for Apache Hive over HTTP).
Example `adbc.hive.conv_data_type=none`. <br><br>Option `scalar` will perform
conversion (if necessary) from the Hiv [...]
-| `adbc.hive.tls_options` | Comma-separated list of TLS/SSL options. Each
option indicates the TLS/SSL option when connecting to a Hive server.
<br><br>Allowed values: `allow_self_signed`, `allow_hostname_mismatch`.
<br><br>Option `allow_self_signed` allows certificate errors due to an unknown
certificate authority, typically when using a self-signed certificate. Option
`allow_hostname_mismatch` allow certificate errors due to a mismatch of the
hostname. (e.g., when connecting through an [...]
| `adbc.hive.connect_timeout_ms` | Sets the timeout (in milliseconds) to open
a new session. Values can be 0 (infinite) or greater than zero. | `30000` |
| `adbc.apache.statement.batch_size` | Sets the maximum number of rows to
retrieve in a single batch request. | `50000` |
| `adbc.apache.statement.polltime_ms` | If polling is necessary to get a
result, this option sets the length of time (in milliseconds) to wait between
polls. | `500` |
| `adbc.apache.statement.query_timeout_s` | Sets the maximum time (in seconds)
for a query to complete. Values can be 0 (infinite) or greater than zero. |
`60` |
+| `adbc.http_options.tls.enabled` | If tls needs to enabled or not. One of
`True`, `False` | `False` |
+| `adbc.http_options.tls.disable_server_certificate_validation` | If tls/ssl
server certificate validation needs to enabled or not. One of `True`, `False`.
If set to True, all certificate validation errors are ignored | `False` |
+| `adbc.http_options.tls.allow_self_signed` | If self signed tls/ssl
certificate needs to be allowed or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.allow_hostname_mismatch` | If hostname mismatch is
allowed for ssl. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.trusted_certificate_path` | The full path of the
tls/ssl certificate .pem file containing custom CA certificates for verifying
the server when connecting over TLS | `` |
## Timeout Configuration
diff --git a/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
b/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
index 7de129df7..914ba9269 100644
--- a/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Impala/ImpalaHttpConnection.cs
@@ -105,7 +105,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(ImpalaParameters.Path, out string? path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port,
ImpalaParameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port,
ImpalaParameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -113,8 +113,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
{
Properties.TryGetValue(ImpalaParameters.DataTypeConv, out string?
dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(ImpalaParameters.TLSOptions, out string?
tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(ImpalaParameters.ConnectTimeoutMilliseconds, out string?
connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -122,6 +120,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
? connectTimeoutMsValue
: throw new
ArgumentOutOfRangeException(ImpalaParameters.ConnectTimeoutMilliseconds,
connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 ..
{int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
internal override IArrowArrayStream NewReader<T>(T statement, Schema
schema) => new HiveServer2Reader(statement, schema, dataTypeConversion:
statement.Connection.DataTypeConversion);
@@ -141,10 +140,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
ImpalaParameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
ImpalaParameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue =
GetAuthenticationHeaderValue(authTypeValue, username, password);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler =
HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
httpClient.DefaultRequestHeaders.Authorization =
authenticationHeaderValue;
@@ -164,24 +163,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback =
(request, certificate, chain, policyErrors) =>
- {
-
- if
(policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) &&
!TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned)) return false;
- if
(policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) &&
!TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch)) return false;
-
- return true;
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue?
GetAuthenticationHeaderValue(ImpalaAuthType authType, string? username, string?
password)
{
if (!string.IsNullOrEmpty(username) &&
!string.IsNullOrEmpty(password) && (authType == ImpalaAuthType.Empty ||
authType == ImpalaAuthType.Basic))
diff --git a/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
b/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
index 43179b625..99ac368be 100644
--- a/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
+++ b/csharp/src/Drivers/Apache/Impala/ImpalaStandardConnection.cs
@@ -96,7 +96,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Impala
Properties.TryGetValue(ImpalaParameters.DataTypeConv, out string?
dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
Properties.TryGetValue(ImpalaParameters.TLSOptions, out string?
tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
}
protected override TTransport CreateTransport()
diff --git a/csharp/src/Drivers/Apache/Spark/README.md
b/csharp/src/Drivers/Apache/Spark/README.md
index 1510c0291..5299a21e8 100644
--- a/csharp/src/Drivers/Apache/Spark/README.md
+++ b/csharp/src/Drivers/Apache/Spark/README.md
@@ -36,11 +36,15 @@ but can also be passed in the call to
`AdbcDatabase.Connect`.
| `username` | The user name used for basic authentication | |
| `password` | The password for the user name used for basic
authentication. | |
| `adbc.spark.data_type_conv` | Comma-separated list of data conversion
options. Each option indicates the type of conversion to perform on data
returned from the Spark server. <br><br>Allowed values: `none`, `scalar`.
<br><br>Option `none` indicates there is no conversion from Spark type to
native type (i.e., no conversion from String to Timestamp for Apache Spark over
HTTP). Example `adbc.spark.conv_data_type=none`. <br><br>Option `scalar` will
perform conversion (if necessary) from th [...]
-| `adbc.spark.tls_options` | Comma-separated list of TLS/SSL options. Each
option indicates the TLS/SSL option when connecting to a Spark server.
<br><br>Allowed values: `allow_self_signed`, `allow_hostname_mismatch`.
<br><br>Option `allow_self_signed` allows certificate errors due to an unknown
certificate authority, typically when using a self-signed certificate. Option
`allow_hostname_mismatch` allow certificate errors due to a mismatch of the
hostname. (e.g., when connecting through [...]
| `adbc.spark.connect_timeout_ms` | Sets the timeout (in milliseconds) to open
a new session. Values can be 0 (infinite) or greater than zero. | `30000` |
| `adbc.apache.statement.batch_size` | Sets the maximum number of rows to
retrieve in a single batch request. | `50000` |
| `adbc.apache.statement.polltime_ms` | If polling is necessary to get a
result, this option sets the length of time (in milliseconds) to wait between
polls. | `500` |
| `adbc.apache.statement.query_timeout_s` | Sets the maximum time (in seconds)
for a query to complete. Values can be 0 (infinite) or greater than zero. |
`60` |
+| `adbc.http_options.tls.enabled` | If tls needs to enabled or not. One of
`True`, `False` | `False` |
+| `adbc.http_options.tls.disable_server_certificate_validation` | If tls/ssl
server certificate validation needs to enabled or not. One of `True`, `False`.
If set to True, all certificate validation errors are ignored | `False` |
+| `adbc.http_options.tls.allow_self_signed` | If self signed tls/ssl
certificate needs to be allowed or not. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.allow_hostname_mismatch` | If hostname mismatch is
allowed for ssl. One of `True`, `False` | `False` |
+| `adbc.http_options.tls.trusted_certificate_path` | The full path of the
tls/ssl certificate .pem file containing custom CA certificates for verifying
the server when connecting over TLS | `` |
## Timeout Configuration
diff --git a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
index a586f4b55..e28ab4632 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkHttpConnection.cs
@@ -121,7 +121,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
Properties.TryGetValue(SparkParameters.Path, out string? path);
_ = new HttpClient()
{
- BaseAddress = GetBaseAddress(uri, hostName, path, port,
SparkParameters.HostName)
+ BaseAddress = GetBaseAddress(uri, hostName, path, port,
SparkParameters.HostName, TlsOptions.IsTlsEnabled)
};
}
@@ -129,8 +129,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
{
Properties.TryGetValue(SparkParameters.DataTypeConv, out string?
dataTypeConv);
DataTypeConversion = DataTypeConversionParser.Parse(dataTypeConv);
- Properties.TryGetValue(SparkParameters.TLSOptions, out string?
tlsOptions);
- TlsOptions = TlsOptionsParser.Parse(tlsOptions);
Properties.TryGetValue(SparkParameters.ConnectTimeoutMilliseconds,
out string? connectTimeoutMs);
if (connectTimeoutMs != null)
{
@@ -138,6 +136,7 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
? connectTimeoutMsValue
: throw new
ArgumentOutOfRangeException(SparkParameters.ConnectTimeoutMilliseconds,
connectTimeoutMs, $"must be a value of 0 (infinite) or between 1 ..
{int.MaxValue}. default is 30000 milliseconds.");
}
+ TlsOptions = HiveServer2TlsImpl.GetHttpTlsOptions(Properties);
}
internal override IArrowArrayStream NewReader<T>(T statement, Schema
schema) => new HiveServer2Reader(statement, schema, dataTypeConversion:
statement.Connection.DataTypeConversion);
@@ -159,10 +158,10 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
Properties.TryGetValue(AdbcOptions.Password, out string? password);
Properties.TryGetValue(AdbcOptions.Uri, out string? uri);
- Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
SparkParameters.HostName);
+ Uri baseAddress = GetBaseAddress(uri, hostName, path, port,
SparkParameters.HostName, TlsOptions.IsTlsEnabled);
AuthenticationHeaderValue? authenticationHeaderValue =
GetAuthenticationHeaderValue(authTypeValue, token, username, password,
access_token);
- HttpClientHandler httpClientHandler = NewHttpClientHandler();
+ HttpClientHandler httpClientHandler =
HiveServer2TlsImpl.NewHttpClientHandler(TlsOptions);
HttpClient httpClient = new(httpClientHandler);
httpClient.BaseAddress = baseAddress;
httpClient.DefaultRequestHeaders.Authorization =
authenticationHeaderValue;
@@ -182,24 +181,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
return transport;
}
- private HttpClientHandler NewHttpClientHandler()
- {
- HttpClientHandler httpClientHandler = new();
- if (TlsOptions != HiveServer2TlsOption.Empty)
- {
- httpClientHandler.ServerCertificateCustomValidationCallback =
(request, certificate, chain, policyErrors) =>
- {
- if (policyErrors == SslPolicyErrors.None) return true;
-
- return
-
(!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateChainErrors) ||
TlsOptions.HasFlag(HiveServer2TlsOption.AllowSelfSigned))
- &&
(!policyErrors.HasFlag(SslPolicyErrors.RemoteCertificateNameMismatch) ||
TlsOptions.HasFlag(HiveServer2TlsOption.AllowHostnameMismatch));
- };
- }
-
- return httpClientHandler;
- }
-
private static AuthenticationHeaderValue?
GetAuthenticationHeaderValue(SparkAuthType authType, string? token, string?
username, string? password, string? access_token)
{
if (!string.IsNullOrEmpty(token) && (authType ==
SparkAuthType.Empty || authType == SparkAuthType.Token))
diff --git a/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
b/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
index d3bd9be6d..8e75ae3f5 100644
--- a/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
+++ b/csharp/src/Drivers/Apache/Spark/SparkParameters.cs
@@ -32,7 +32,6 @@ namespace Apache.Arrow.Adbc.Drivers.Apache.Spark
public const string AuthType = "adbc.spark.auth_type";
public const string Type = "adbc.spark.type";
public const string DataTypeConv = "adbc.spark.data_type_conv";
- public const string TLSOptions = "adbc.spark.tls_options";
public const string ConnectTimeoutMilliseconds =
"adbc.spark.connect_timeout_ms";
}
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
b/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
index 992e5ffb1..b481080bc 100644
--- a/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2ParametersTest.cs
@@ -34,16 +34,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
Assert.Throws(exceptionType, () =>
DataTypeConversionParser.Parse(dataTypeConversion));
}
- [SkippableTheory]
- [MemberData(nameof(GetParametersTlsOptionTestData))]
- internal void TestParametersTlsOptionParse(string? tlsOptions,
HiveServer2TlsOption expected, Type? exceptionType = default)
- {
- if (exceptionType == default)
- Assert.Equal(expected, TlsOptionsParser.Parse(tlsOptions));
- else
- Assert.Throws(exceptionType, () =>
TlsOptionsParser.Parse(tlsOptions));
- }
-
public static IEnumerable<object?[]>
GetParametersDataTypeConvTestData()
{
// Default
@@ -68,26 +58,5 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
yield return new object?[] { $"xxx", DataTypeConversion.Empty,
typeof(ArgumentOutOfRangeException) };
yield return new object?[] { $"none,scalar,xxx",
DataTypeConversion.None | DataTypeConversion.Scalar,
typeof(ArgumentOutOfRangeException) };
}
-
- public static IEnumerable<object?[]> GetParametersTlsOptionTestData()
- {
- // Default
- yield return new object?[] { null, HiveServer2TlsOption.Empty };
- yield return new object?[] { "", HiveServer2TlsOption.Empty};
- yield return new object?[] { " ", HiveServer2TlsOption.Empty };
- // Explicit
- yield return new object?[] { $"{TlsOptions.AllowSelfSigned}",
HiveServer2TlsOption.AllowSelfSigned };
- yield return new object?[] {
$"{TlsOptions.AllowHostnameMismatch}",
HiveServer2TlsOption.AllowHostnameMismatch };
- // Ignore empty
- yield return new object?[] { $",{TlsOptions.AllowSelfSigned}",
HiveServer2TlsOption.AllowSelfSigned };
- yield return new object?[] {
$",{TlsOptions.AllowHostnameMismatch},",
HiveServer2TlsOption.AllowHostnameMismatch };
- // Combined, embedded space, mixed-case
- yield return new object?[] {
$"{TlsOptions.AllowSelfSigned},{TlsOptions.AllowHostnameMismatch}",
HiveServer2TlsOption.AllowSelfSigned |
HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] {
$"{TlsOptions.AllowHostnameMismatch},{TlsOptions.AllowSelfSigned}",
HiveServer2TlsOption.AllowSelfSigned |
HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] { $" {TlsOptions.AllowHostnameMismatch}
, {TlsOptions.AllowSelfSigned} ", HiveServer2TlsOption.AllowSelfSigned |
HiveServer2TlsOption.AllowHostnameMismatch };
- yield return new object?[] {
$"{TlsOptions.AllowSelfSigned.ToUpperInvariant()},{TlsOptions.AllowHostnameMismatch.ToUpperInvariant()}",
HiveServer2TlsOption.AllowSelfSigned |
HiveServer2TlsOption.AllowHostnameMismatch };
- // Invalid
- yield return new object?[] {
$"xxx,{TlsOptions.AllowSelfSigned.ToUpperInvariant()},{TlsOptions.AllowHostnameMismatch.ToUpperInvariant()}",
HiveServer2TlsOption.Empty, typeof(ArgumentOutOfRangeException) };
- }
}
}
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
index 79829ae40..c442a8a7e 100644
--- a/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2TestEnvironment.cs
@@ -90,10 +90,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
{
parameters.Add(HiveServer2Parameters.DataTypeConv,
testConfiguration.DataTypeConversion!);
}
- if (!string.IsNullOrEmpty(testConfiguration.TlsOptions))
- {
- parameters.Add(HiveServer2Parameters.TLSOptions,
testConfiguration.TlsOptions!);
- }
if (!string.IsNullOrEmpty(testConfiguration.BatchSize))
{
parameters.Add(ApacheParameters.BatchSize,
testConfiguration.BatchSize!);
diff --git a/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs
b/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs
new file mode 100644
index 000000000..70d5534e6
--- /dev/null
+++ b/csharp/test/Drivers/Apache/Hive2/HiveServer2TlsImplTest.cs
@@ -0,0 +1,59 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Apache.Arrow.Adbc.Drivers.Apache.Hive2;
+using Xunit;
+
+namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Hive2
+{
+ public class HiveServer2TlsImplTest
+ {
+ [SkippableTheory]
+ [MemberData(nameof(GetSslOptionsTestData))]
+ internal void TestValidateTlsOptions(Dictionary<string, string>?
dataTypeConversion, TlsProperties expected, Type? exceptionType = default)
+ {
+ if (exceptionType == default)
+ Assert.Equivalent(expected,
HiveServer2TlsImpl.GetHttpTlsOptions(dataTypeConversion ?? new
Dictionary<string, string>()));
+ else
+ Assert.Throws(exceptionType, () =>
HiveServer2TlsImpl.GetHttpTlsOptions(dataTypeConversion ?? new
Dictionary<string, string>()));
+ }
+
+ public static IEnumerable<object?[]> GetSslOptionsTestData()
+ {
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "False" } }, new TlsProperties { IsTlsEnabled =
false } };
+ yield return new object?[] { new Dictionary<string, string> { {
AdbcOptions.Uri, "https://arrow.apache.org"; } }, new TlsProperties {
IsTlsEnabled = true, DisableServerCertificateValidation = false } };
+ // uri takes precedence over ssl option
+ yield return new object?[] { new Dictionary<string, string> { {
AdbcOptions.Uri, "https://arrow.apache.org"; }, { HttpTlsOptions.IsTlsEnabled,
"False" } }, new TlsProperties { IsTlsEnabled = true,
DisableServerCertificateValidation = false } };
+ // other ssl options are ignored if
disableServerCertificateValidation is set to true
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "True" }, {
HttpTlsOptions.DisableServerCertificateValidation, "True" }, {
HttpTlsOptions.AllowSelfSigned, "True" }, {
HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties {
IsTlsEnabled = true, DisableServerCertificateValidation = true } };
+ // other ssl options are ignored if ssl is disabled
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "False" }, { HttpTlsOptions.AllowSelfSigned,
"True" }, { HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties
{ IsTlsEnabled = false } };
+ // case insensitive boolean string parsing
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "false" } }, new TlsProperties { IsTlsEnabled =
false } };
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "True" } }, new TlsProperties { IsTlsEnabled =
true, DisableServerCertificateValidation = false, AllowSelfSigned = false,
AllowHostnameMismatch = false } };
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "tRUe" }, { HttpTlsOptions.AllowSelfSigned, "true"
} }, new TlsProperties { IsTlsEnabled = true,
DisableServerCertificateValidation = false, AllowSelfSigned = true,
AllowHostnameMismatch = false } };
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "TruE" }, { HttpTlsOptions.AllowSelfSigned, "True"
}, { HttpTlsOptions.AllowHostnameMismatch, "True" } }, new TlsProperties {
IsTlsEnabled = true, DisableServerCertificateValidation = false,
AllowSelfSigned = true, AllowHostnameMismatch = true } };
+ // certificate path is ignored if self signed is not allowed
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "True" }, { HttpTlsOptions.AllowSelfSigned,
"False" }, { HttpTlsOptions.AllowHostnameMismatch, "True" }, {
HttpTlsOptions.TrustedCertificatePath, "" } }, new TlsProperties { IsTlsEnabled
= true, DisableServerCertificateValidation = false, AllowSelfSigned = false,
AllowHostnameMismatch = true } };
+ // invalid certificate path
+ yield return new object?[] { new Dictionary<string, string> { {
HttpTlsOptions.IsTlsEnabled, "True" }, { HttpTlsOptions.AllowSelfSigned, "True"
}, { HttpTlsOptions.AllowHostnameMismatch, "True" }, {
HttpTlsOptions.TrustedCertificatePath, "" } }, null,
typeof(FileNotFoundException) };
+ }
+ }
+}
diff --git a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
index 9c9124d6d..265ac6b96 100644
--- a/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
+++ b/csharp/test/Drivers/Apache/Spark/SparkTestEnvironment.cs
@@ -101,10 +101,6 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.Apache.Spark
{
parameters.Add(SparkParameters.DataTypeConv,
testConfiguration.DataTypeConversion!);
}
- if (!string.IsNullOrEmpty(testConfiguration.TlsOptions))
- {
- parameters.Add(SparkParameters.TLSOptions,
testConfiguration.TlsOptions!);
- }
if (!string.IsNullOrEmpty(testConfiguration.BatchSize))
{
parameters.Add(ApacheParameters.BatchSize,
testConfiguration.BatchSize!);
