This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 2bd19f726 BIGTOP-3900. Migrate secure DataNode settings from jsvc to
SASL. (#1083)
2bd19f726 is described below
commit 2bd19f726d826cbf340fca6a6665bd357822bc91
Author: Masatake Iwasaki <[email protected]>
AuthorDate: Mon Feb 6 12:19:58 2023 +0900
BIGTOP-3900. Migrate secure DataNode settings from jsvc to SASL. (#1083)
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 23 +++---
.../puppet/modules/hadoop/files/http.keystore | Bin 0 -> 2246 bytes
.../puppet/modules/hadoop/files/http.truststore | Bin 0 -> 2246 bytes
.../puppet/modules/hadoop/manifests/init.pp | 45 ++++++++++-
.../puppet/modules/hadoop/templates/core-site.xml | 8 ++
.../puppet/modules/hadoop/templates/hadoop-hdfs | 6 --
.../puppet/modules/hadoop/templates/hdfs-site.xml | 12 +--
.../puppet/modules/hadoop/templates/ssl-client.xml | 80 +++++++++++++++++++
.../puppet/modules/hadoop/templates/ssl-server.xml | 88 +++++++++++++++++++++
9 files changed, 236 insertions(+), 26 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 1895eef85..56e3abaa6 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -79,6 +79,19 @@
# applies to hdfs, yarn, mapred, kms and httpfs
hadoop::kerberos_realm: "%{hiera('kerberos::krb_site::realm')}"
+# skip hostname verification for testing deployment.
+hadoop::hadoop_ssl_hostname_verifier: "ALLOW_ALL"
+
+# Set as shown below in site.yaml to also enable Kerberos authentication
+# on the web GUIs of journalnode, namenode, datanode, resourcemanager and
+# nodemanager when you enable Kerberos for Hadoop API communication. This
+# intentionally is not the default right now.
+#hadoop::hadoop_http_authentication_type:
"%{hiera('hadoop::hadoop_security_authentication')}"
+
+# A secret is necessary for the cross-service-cross-node session cookie.
+# Provide this by setting the following to something long and secret:
+#hadoop::hadoop_http_authentication_signature_secret: changeit
+
hadoop::common_hdfs::hadoop_namenode_host:
"%{hiera('bigtop::hadoop_head_node')}"
# actually default but needed for hadoop_namenode_uri here
hadoop::common_hdfs::hadoop_namenode_port: "8020"
@@ -96,16 +109,6 @@ hadoop::common_hdfs::hadoop_namenode_port: "8020"
hadoop::common_hdfs::hdfs_shortcut_reader: true
-# Set as shown below in site.yaml to also enable Kerberos authentication
-# on the web GUIs of journalnode, namenode, datanode, resourcemanager and
-# nodemanager when you enable Kerberos for Hadoop API communication. This
-# intentionally is not the default right now.
-#hadoop::common_hdfs::hadoop_http_authentication_type:
"%{hiera('hadoop::hadoop_security_authentication')}"
-#
-# A secret is necessary for the cross-service-cross-node session cookie.
-# Provide this by setting the following to something long and secret:
-#hadoop::common_hdfs::hadoop_http_authentication_signature_secret:
-
# ... or have trocla generate secrets for Hadoop components automatically. For
# this to work, the trocla puppet module must be installed.
#hadoop::generate_secrets: true
diff --git a/bigtop-deploy/puppet/modules/hadoop/files/http.keystore
b/bigtop-deploy/puppet/modules/hadoop/files/http.keystore
new file mode 100644
index 000000000..9af4fe5d2
Binary files /dev/null and
b/bigtop-deploy/puppet/modules/hadoop/files/http.keystore differ
diff --git a/bigtop-deploy/puppet/modules/hadoop/files/http.truststore
b/bigtop-deploy/puppet/modules/hadoop/files/http.truststore
new file mode 100644
index 000000000..9af4fe5d2
Binary files /dev/null and
b/bigtop-deploy/puppet/modules/hadoop/files/http.truststore differ
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 4a9b72b86..c7a161313 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -26,6 +26,11 @@ class hadoop ($hadoop_security_authentication = "simple",
$generate_secrets = false,
$kms_host = undef,
$kms_port = undef,
+ $hadoop_ssl_hostname_verifier = undef,
+ $hadoop_http_authentication_type = undef,
+ $hadoop_http_authentication_signature_secret = undef,
+ $hadoop_http_authentication_signature_secret_file =
"/etc/hadoop/conf/hadoop-http-authentication-signature-secret",
+ $hadoop_http_authentication_cookie_domain = regsubst($fqdn, "^[^\\.]+\\.",
""),
) {
include stdlib
@@ -236,10 +241,6 @@ class hadoop ($hadoop_security_authentication = "simple",
$hadoop_snappy_codec = undef,
$hadoop_security_authentication =
$hadoop::hadoop_security_authentication,
$kerberos_realm = $hadoop::kerberos_realm,
- $hadoop_http_authentication_type = undef,
- $hadoop_http_authentication_signature_secret = undef,
- $hadoop_http_authentication_signature_secret_file =
"/etc/hadoop/conf/hadoop-http-authentication-signature-secret",
- $hadoop_http_authentication_cookie_domain = regsubst($fqdn,
"^[^\\.]+\\.", ""),
$generate_secrets = $hadoop::generate_secrets,
$namenode_datanode_registration_ip_hostname_check = undef,
$kms_host = $hadoop::kms_host,
@@ -319,6 +320,42 @@ class hadoop ($hadoop_security_authentication = "simple",
require => [Package["hadoop"]],
}
+ file {
+ "/etc/hadoop/conf/ssl-client.xml":
+ content => template('hadoop/ssl-client.xml'),
+ owner => 'root',
+ group => 'hadoop',
+ mode => '0660',
+ require => [Package["hadoop"]],
+ }
+
+ file {
+ "/etc/hadoop/conf/ssl-server.xml":
+ content => template('hadoop/ssl-server.xml'),
+ owner => 'root',
+ group => 'hadoop',
+ mode => '0660',
+ require => [Package["hadoop"]],
+ }
+
+ file {
+ "/etc/hadoop/conf/http.keystore":
+ source => "puppet:///modules/hadoop/http.keystore",
+ owner => 'root',
+ group => 'hadoop',
+ mode => '0660',
+ require => [Package["hadoop"]],
+ }
+
+ file {
+ "/etc/hadoop/conf/http.truststore":
+ source => "puppet:///modules/hadoop/http.truststore",
+ owner => 'root',
+ group => 'hadoop',
+ mode => '0660',
+ require => [Package["hadoop"]],
+ }
+
if $hadoop_http_authentication_type == "kerberos" {
if $generate_secrets {
$http_auth_sig_secret =
trocla("hadoop_http_authentication_signature_secret", "plain")
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
index 1cae18da8..f4398d802 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
@@ -164,4 +164,12 @@
<value>kms://http@<%= @kms_host %>:<%= @kms_port %>/kms</value>
</property>
<% end -%>
+
+<% if @hadoop_ssl_hostname_verifier %>
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value><%= @hadoop_ssl_hostname_verifier %></value>
+ </property>
+<% end -%>
+
</configuration>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs
b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs
index 04572dd62..2a841591e 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs
@@ -18,9 +18,3 @@ export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
export HADOOP_DATANODE_USER=hdfs
export HADOOP_IDENT_STRING=hdfs
-
-<% if @hadoop_security_authentication == "kerberos" %>
-export HADOOP_SECURE_DN_USER=hdfs
-export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PID_DIR
-export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_LOG_DIR
-<% end %>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
index 3d10c27cc..c3b805e30 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
@@ -151,6 +151,10 @@
<name>dfs.block.access.token.enable</name>
<value>true</value>
</property>
+ <property>
+ <name>dfs.http.policy</name>
+ <value>HTTPS_ONLY</value>
+ </property>
<!-- NameNode security config -->
<property>
@@ -215,12 +219,8 @@
<value>700</value>
</property>
<property>
- <name>dfs.datanode.address</name>
- <value>0.0.0.0:1004</value>
- </property>
- <property>
- <name>dfs.datanode.http.address</name>
- <value>0.0.0.0:1006</value>
+ <name>dfs.data.transfer.protection</name>
+ <value>privacy</value>
</property>
<property>
<name>dfs.datanode.keytab.file</name>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/ssl-client.xml
b/bigtop-deploy/puppet/modules/hadoop/templates/ssl-client.xml
new file mode 100644
index 000000000..c8dd54e6f
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/ssl-client.xml
@@ -0,0 +1,80 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.client.truststore.location</name>
+ <value>/etc/hadoop/conf/http.truststore</value>
+ <description>Truststore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.password</name>
+ <value>changeit</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.location</name>
+ <value>/etc/hadoop/conf/http.keystore</value>
+ <description>Keystore to be used by clients like distcp. Must be
+ specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.password</name>
+ <value>changeit</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.keypassword</name>
+ <value>changeit</value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+</configuration>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/ssl-server.xml
b/bigtop-deploy/puppet/modules/hadoop/templates/ssl-server.xml
new file mode 100644
index 000000000..d884f7616
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/ssl-server.xml
@@ -0,0 +1,88 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+<property>
+ <name>ssl.server.truststore.location</name>
+ <value></value>
+ <description>Truststore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.password</name>
+ <value></value>
+ <description>Optional. Default value is "".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ <description>Truststore reload check interval, in milliseconds.
+ Default value is 10000 (10 seconds).
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.location</name>
+ <value>/etc/hadoop/conf/http.keystore</value>
+ <description>Keystore to be used by NN and DN. Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.password</name>
+ <value>changeit</value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.keypassword</name>
+ <value>changeit</value>
+ <description>Must be specified.
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ <description>Optional. The keystore file format, default value is "jks".
+ </description>
+</property>
+
+<property>
+ <name>ssl.server.exclude.cipher.list</name>
+ <value>TLS_ECDHE_RSA_WITH_RC4_128_SHA,SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA,
+ SSL_RSA_WITH_DES_CBC_SHA,SSL_DHE_RSA_WITH_DES_CBC_SHA,
+ SSL_RSA_EXPORT_WITH_RC4_40_MD5,SSL_RSA_EXPORT_WITH_DES40_CBC_SHA,
+ SSL_RSA_WITH_RC4_128_MD5</value>
+ <description>Optional. The weak security cipher suites that you want excluded
+ from SSL communication.</description>
+</property>
+
+</configuration>
