Repository: ambari Updated Branches: refs/heads/trunk 9371bb760 -> 8e3433c82
AMBARI-11289. Ranger HDFS/HBase Plugin Upgrade Pack For HDP-2.2 To HDP-2.3 (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8e3433c8 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8e3433c8 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8e3433c8 Branch: refs/heads/trunk Commit: 8e3433c82063a0308596db37a940a6e264ca45de Parents: 9371bb7 Author: Nate Cole <[email protected]> Authored: Wed May 20 20:12:36 2015 -0400 Committer: Nate Cole <[email protected]> Committed: Thu May 21 13:29:30 2015 -0400 ---------------------------------------------------------------------- .../serveraction/upgrades/ConfigureAction.java | 14 ++- .../state/stack/upgrade/ConfigureTask.java | 17 ++- .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 112 ++++++++++++++++++- .../upgrades/ConfigureActionTest.java | 4 + 4 files changed, 140 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java index c5f33c4..69a03f5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java @@ -275,7 +275,7 @@ public class ConfigureAction extends AbstractServerAction { // append standard output outputBuffer.append(MessageFormat.format("Created {0}/{1} = {2}\n", configType, - transfer.toKey, valueToCopy)); + transfer.toKey, mask(transfer, valueToCopy))); } break; case MOVE: @@ -295,7 +295,7 @@ public class ConfigureAction extends AbstractServerAction { // append standard output outputBuffer.append(MessageFormat.format("Created {0}/{1} with default value {2}\n", - configType, transfer.toKey, transfer.defaultValue)); + configType, transfer.toKey, mask(transfer, transfer.defaultValue))); } break; @@ -370,7 +370,7 @@ public class ConfigureAction extends AbstractServerAction { // without a key/value to set newValues.put(key, value); outputBuffer.append(MessageFormat.format("{0}/{1} changed to {2}\n", configType, key, - value)); + mask(keyValuePair, value))); } } } @@ -506,4 +506,12 @@ public class ConfigureAction extends AbstractServerAction { return result; } + + private static String mask(ConfigureTask.Masked mask, String value) { + if (mask.mask) { + return StringUtils.repeat("*", value.length()); + } + return value; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java index 99d6058..f5a77c5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java @@ -154,11 +154,22 @@ public class ConfigureTask extends ServerSideActionTask { } /** + * Used for configuration updates that should mask their values from being + * printed in plain text. + */ + @XmlAccessorType(XmlAccessType.FIELD) + public static class Masked { + @XmlAttribute(name = "mask") + public boolean mask = false; + } + + + /** * A key/value pair to set in the type specified by {@link ConfigureTask#type} */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "set") - public static class ConfigurationKeyValue { + public static class ConfigurationKeyValue extends Masked { @XmlAttribute(name = "key") public String key; @@ -197,7 +208,7 @@ public class ConfigureTask extends ServerSideActionTask { */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "transfer") - public static class Transfer { + public static class Transfer extends Masked { /** * The type of operation, such as COPY or DELETE. */ @@ -291,7 +302,7 @@ public class ConfigureTask extends ServerSideActionTask { */ @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "replace") - public static class Replace { + public static class Replace extends Masked { /** * The key name */ http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml index 7505a07..ddaee86 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml @@ -90,7 +90,6 @@ <service name="HBASE"> <component>HBASE_MASTER</component> </service> - </group> <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check"> @@ -332,6 +331,60 @@ <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value> </condition> </task> + <task xsi:type="configure" summary="Transitioning Ranger HDFS Policy"> + <type>ranger-hdfs-policymgr-ssl</type> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit" /> + </task> + <task xsi:type="configure" summary="Transitioning Ranger HDFS Audit"> + <type>ranger-hdfs-audit</type> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED" to-key="xasecure.audit.destination.db" default-value="false"/> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/log/hadoop/hdfs/audit/hdfs/spool" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value="" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" mask="true" default-value="" /> + <set key="xasecure.audit.destination.solr" value="true" /> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}" /> + <set key="xasecure.audit.destination.solr.zookeepers" value="" /> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hdfs/audit/solr/spool" /> + <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}" /> + <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}" /> + <set key="xasecure.audit.provider.summary.enabled" value="true" /> + </task> + + <task xsi:type="configure" summary="Transitioning Ranger HDFS Security"> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="REPOSITORY_NAME" to-key="ranger.plugin.hdfs.service.name" default-value="{{repo_name}}" /> + <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="POLICY_MGR_URL" to-key="ranger.plugin.hdfs.policy.rest.url" default-value="{{policymgr_mgr_url}}" /> + </task> + + <task xsi:type="configure"> + <type>ranger-hdfs-plugin-properties</type> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" /> + <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" /> + <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" /> + <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" /> + <transfer operation="delete" delete-key="REPOSITORY_NAME" /> + <transfer operation="delete" delete-key="POLICY_MGR_URL" /> + </task> </pre-upgrade> <upgrade> <task xsi:type="restart" /> @@ -427,6 +480,63 @@ <service name="HBASE"> <component name="HBASE_MASTER"> + <pre-upgrade> + <task xsi:type="configure" summary="Transitioning Ranger HBase Policy"> + <type>ranger-hbase-policymgr-ssl</type> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-keystore.jks" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH" to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-truststore.jks" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD" to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit" /> + </task> + <task xsi:type="configure" summary="Transitioning Ranger HBase Audit"> + <type>ranger-hbase-audit</type> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED" to-key="xasecure.audit.destination.db" default-value="false" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED" to-key="xasecure.audit.destination.hdfs" default-value="true" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/logs/hadoop/hdfs/audit/hdfs/spool" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.USER_NAME" to-key="xasecure.audit.destination.db.user" default-value="" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.PASSWORD" to-key="xasecure.audit.destination.db.password" mask="true" default-value="" /> + <set key="xasecure.audit.destination.solr" value="true" /> + <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}" /> + <set key="xasecure.audit.destination.solr.zookeepers" value="" /> + <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hbase/audit/solr/spool" /> + <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}" /> + <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}" /> + <set key="xasecure.audit.provider.summary.enabled" value="true" /> + </task> + <task xsi:type="configure"> + <type>ranger-hbase-security</type> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE" to-key="xasecure.hbase.update.xapolicies.on.grant.revoke" default-value="true" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="POLICY_MGR_URL" to-key="ranger.plugin.hbase.policy.rest.url" default-value="{{policymgr_mgr_url}}" /> + <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="REPOSITORY_NAME" to-key="ranger.plugin.hbase.service.name" default-value="{{repo_name}}" /> + </task> + <task xsi:type="configure"> + <type>ranger-hbase-plugin-properties</type> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS" /> + <transfer operation="delete" delete-key="SQL_CONNECTOR_JAR" /> + <transfer operation="delete" delete-key="REPOSITORY_NAME" /> + <transfer operation="delete" delete-key="POLICY_MGR_URL" /> + <transfer operation="delete" delete-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE" /> + <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" /> + <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" /> + <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" /> + <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" /> + <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" /> + <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" /> + </task> + </pre-upgrade> <upgrade> <task xsi:type="restart" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java index c2ea948..08a8c35 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java @@ -306,6 +306,7 @@ public class ConfigureActionTest { transfer.fromKey = "movedFromKeyMissingWithDefault"; transfer.toKey = "movedToMissingWithDefault"; transfer.defaultValue = "defaultValue2"; + transfer.mask = true; transfers.add(transfer); transfer = new ConfigureTask.Transfer(); @@ -544,6 +545,7 @@ public class ConfigureActionTest { configurations.add(fooKey3); fooKey3.key = "fooKey3"; fooKey3.value = "barValue3"; + fooKey3.mask = true; Map<String, String> commandParams = new HashMap<String, String>(); commandParams.put("upgrade_direction", "upgrade"); @@ -573,6 +575,8 @@ public class ConfigureActionTest { assertEquals("barValue", config.getProperties().get("fooKey")); assertEquals("barValue2", config.getProperties().get("fooKey2")); assertEquals("barValue3", config.getProperties().get("fooKey3")); + assertTrue(report.getStdOut().contains("******")); + } private void makeUpgradeCluster() throws Exception {
