Repository: ambari Updated Branches: refs/heads/branch-2.4 116d47df8 -> c0f4d1b6f
AMBARI-16961. Remove LogSearch dependency from ATLAS (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c0f4d1b6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c0f4d1b6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c0f4d1b6 Branch: refs/heads/branch-2.4 Commit: c0f4d1b6f317a65a7b8812984789d959eb0d44d0 Parents: 116d47d Author: oleewere <[email protected]> Authored: Thu Jun 2 01:35:41 2016 +0200 Committer: oleewere <[email protected]> Committed: Thu Jun 2 02:24:36 2016 +0200 ---------------------------------------------------------------------- .../libraries/functions/package_conditions.py | 4 +- .../libraries/functions/solr_cloud_util.py | 40 +++++++++++++++++++- .../ATLAS/0.1.0.2.3/metainfo.xml | 4 ++ .../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 10 +++-- .../ATLAS/0.1.0.2.3/package/scripts/params.py | 4 ++ .../package/scripts/setup_logsearch_solr.py | 31 ++------------- .../stacks/HDP/2.3/role_command_order.json | 2 +- .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 36 +++++++++++++----- .../stacks/2.3/ATLAS/test_metadata_server.py | 34 ++++++++++++++++- .../test/python/stacks/2.3/configs/default.json | 8 +++- .../test/python/stacks/2.3/configs/secure.json | 8 +++- .../test/python/stacks/2.4/configs/default.json | 3 ++ .../stacks/2.5/ATLAS/test_atlas_server.py | 33 ++++++++++++++++ .../test/python/stacks/2.5/configs/default.json | 8 +++- 14 files changed, 176 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py index b916f19..ecc4f49 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py @@ -52,7 +52,7 @@ def should_install_logsearch_solr(): def should_install_logsearch_solr_client(): config = Script.get_config() - return 'role' in config and config['role'] == "LOGSEARCH_SOLR_CLIENT" + return 'role' in config and (config['role'] == "LOGSEARCH_SOLR_CLIENT" or config['role'] == 'ATLAS_SERVER') def should_install_logsearch_portal(): config = Script.get_config() @@ -89,4 +89,4 @@ def should_install_ranger_tagsync(): def should_install_rpcbind(): config = Script.get_config() - return 'role' in config and config['role'] == "NFS_GATEWAY" \ No newline at end of file + return 'role' in config and config['role'] == "NFS_GATEWAY" http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py index e6e794c..b099a1e 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/solr_cloud_util.py @@ -16,8 +16,10 @@ See the License for the specific language governing permissions and limitations under the License. """ +from resource_management.libraries.functions.default import default from resource_management.libraries.functions.format import format -from resource_management.core.resources.system import Execute +from resource_management.core.resources.system import Directory, Execute, File +from resource_management.core.source import StaticFile, InlineTemplate __all__ = ["upload_configuration_to_zk", "create_collection"] @@ -73,3 +75,39 @@ def create_collection(zookeeper_quorum, solr_znode, collection, config_set, java Execute(create_collection_cmd, user=user, group=group ) + +def setup_solr_client(config, user = None, group = None): + solr_user = config['configurations']['logsearch-solr-env']['logsearch_solr_user'] if user is None else user + solr_group = config['configurations']['logsearch-solr-env']['logsearch_solr_group'] if group is None else group + solr_client_dir = '/usr/lib/ambari-logsearch-solr-client' + solr_client_log_dir = default('/configurations/logsearch-solr-env/logsearch_solr_client_log_dir', '/var/log/ambari-logsearch-solr-client') + solr_client_log = format("{solr_client_log_dir}/solr-client.log") + solr_client_log4j_content = config['configurations']['logsearch-solr-client-log4j']['content'] + + Directory([solr_client_dir, solr_client_log_dir], + mode=0755, + cd_access='a', + owner=solr_user, + group=solr_group, + create_parents=True + ) + solrCliFilename = format("{solr_client_dir}/solrCloudCli.sh") + File(solrCliFilename, + mode=0755, + owner=solr_user, + group=solr_group, + content=StaticFile(solrCliFilename) + ) + File(format("{solr_client_dir}/log4j.properties"), + content=InlineTemplate(solr_client_log4j_content), + owner=solr_user, + group=solr_group, + mode=0644 + ) + + File(solr_client_log, + mode=0644, + owner=solr_user, + group=solr_group, + content='' + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml index d0d73ad..9f3eda9 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml @@ -98,6 +98,10 @@ <package> <name>atlas-metadata</name> </package> + <package> + <name>ambari-logsearch-solr-client</name> + <condition>should_install_logsearch_solr_client</condition> + </package> </packages> </osSpecific> </osSpecifics> http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py index 1a6efe1..5fd6523 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py @@ -108,6 +108,8 @@ def metadata(type='server'): ) if type == 'server' and params.search_backend_solr and params.has_logsearch_solr: + solr_cloud_util.setup_solr_client(params.config) + random_num = random.random() upload_conf_set('basic_configs', random_num) @@ -131,8 +133,8 @@ def upload_conf_set(config_set, random_num): config_set=config_set, tmp_config_set_dir=tmp_config_set_folder, java64_home=params.java64_home, - user=params.metadata_user, - group=params.user_group) + user=params.logsearch_solr_user, + group=params.logsearch_solr_group) def create_collection(collection, config_set): import params @@ -143,7 +145,7 @@ def create_collection(collection, config_set): collection = collection, config_set=config_set, java64_home=params.java64_home, - user=params.metadata_user, - group=params.user_group, + user=params.logsearch_solr_user, + group=params.logsearch_solr_group, shards=params.atlas_solr_shards, replication_factor = params.logsearch_solr_replication_factor) http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index 33e5d9e..a79a456 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -172,6 +172,10 @@ logsearch_solr_replication_factor = 2 if len(logsearch_solr_hosts) > 1 else 1 atlas_solr_shards = default("/configurations/atlas-env/atlas_solr-shards", 1) has_logsearch_solr = len(logsearch_solr_hosts) > 0 +if has_logsearch_solr: + logsearch_solr_user = config['configurations']['logsearch-solr-env']['logsearch_solr_user'] + logsearch_solr_group = config['configurations']['logsearch-solr-env']['logsearch_solr_group'] + # zookeeper zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py index 767e208..6e71334 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch_solr.py @@ -18,10 +18,10 @@ limitations under the License. """ from resource_management.core.exceptions import Fail +from resource_management.core.source import InlineTemplate, Template from resource_management.core.resources.system import Directory, Execute, File -from resource_management.core.source import StaticFile from resource_management.libraries.functions.format import format -from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.functions import solr_cloud_util def setup_logsearch_solr(name = None): @@ -82,32 +82,7 @@ def setup_logsearch_solr(name = None): user=params.logsearch_solr_user ) elif name == 'client': - Directory([params.solr_client_dir, params.logsearch_solr_client_log_dir], - mode=0755, - cd_access='a', - owner=params.logsearch_solr_user, - group=params.logsearch_solr_group, - create_parents=True - ) - solrCliFilename = format("{solr_client_dir}/solrCloudCli.sh") - File(solrCliFilename, - mode=0755, - group=params.logsearch_solr_group, - owner=params.logsearch_solr_user, - content=StaticFile(solrCliFilename) - ) - - File(format("{solr_client_dir}/log4j.properties"), - content=InlineTemplate(params.solr_client_log4j_content), - owner=params.logsearch_solr_user - ) - - File(params.logsearch_solr_client_log, - mode=0644, - owner=params.logsearch_solr_user, - group=params.logsearch_solr_group, - content='' - ) + solr_cloud_util.setup_solr_client(params.config) else : raise Fail('Nor client or server were selected to install.') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json index ce2528b..624decd 100755 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json @@ -9,7 +9,7 @@ "RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"], "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"], "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"], - "ATLAS_SERVER-START": ["KAFKA_BROKER-START"], + "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "LOGSEARCH_SOLR-START"], "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"], "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START", "NODEMANAGER-START"], "HAWQSTANDBY-START" : ["HAWQMASTER-START"], http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 4890f00..195eb7b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -25,15 +25,6 @@ <component> <name>ATLAS_SERVER</name> <cardinality>1+</cardinality> - <dependencies> - <dependency> - <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name> - <scope>host</scope> - <auto-deploy> - <enabled>true</enabled> - </auto-deploy> - </dependency> - </dependencies> <configFile> <type>properties</type> <fileName>atlas-application.properties</fileName> @@ -51,6 +42,33 @@ </component> </components> + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>atlas-metadata_${stack_version}</name> + </package> + <package> + <name>ambari-logsearch-solr-client</name> + <condition>should_install_logsearch_solr_client</condition> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>atlas-metadata-${stack_version}</name> + </package> + <package> + <name>ambari-logsearch-solr-client</name> + <condition>should_install_logsearch_solr_client</condition> + </package> + </packages> + </osSpecific> + </osSpecifics> + <requiredServices> <service>KAFKA</service> </requiredServices> http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py index faa7350..c9a8de0 100644 --- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py +++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py @@ -108,7 +108,39 @@ class TestMetadataServer(RMFTestCase): group='hadoop', mode=0644, ) - + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', + owner='solr', + group='solr', + create_parents = True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', + owner='solr', + group='solr', + create_parents = True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', + content=StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), + owner='solr', + group='solr', + mode=0755, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', + content=InlineTemplate(self.getConfig()['configurations'][ + 'logsearch-solr-client-log4j']['content']), + owner='solr', + group='solr', + mode=0644, + ) + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', + owner='solr', + group='solr', + mode=0644, + content = '' + ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf -cs basic_configs -rt 5 -i 10') http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.3/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json index 157fd7e..07298f1 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json @@ -172,7 +172,13 @@ "log.retention.hours": "168" }, "logsearch-solr-env": { - "logsearch_solr_znode": "/logsearch" + "logsearch_solr_znode": "/logsearch", + "logsearch_solr_user": "solr", + "logsearch_solr_group": "solr", + "logsearch_solr_client_log_dir" :"/var/log/ambari-logsearch-solr-client" + }, + "logsearch-solr-client-log4j" : { + "content" : "content" }, "application-properties": { "atlas.cluster.name" : "c2", http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.3/configs/secure.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json index 3ad8656..1ca7a2e 100644 --- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json +++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json @@ -163,7 +163,13 @@ "log.retention.hours": "168" }, "logsearch-solr-env": { - "logsearch_solr_znode": "/logsearch" + "logsearch_solr_znode": "/logsearch", + "logsearch_solr_user": "solr", + "logsearch_solr_group": "solr", + "logsearch_solr_client_log_dir" :"/var/log/ambari-logsearch-solr-client" + }, + "logsearch-solr-client-log4j" : { + "content" : "content" }, "application-properties": { "atlas.cluster.name" : "c2", http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index 6362a4c..c3aecff 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -229,6 +229,9 @@ "logsearch_admin_password" : "admin", "content": "admin json content" }, + "logsearch-solr-client-log4j" : { + "content" : "content" + }, "logsearch-solr-env": { "logsearch_solr_user": "solr", "logsearch_solr_group": "solr", http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py index 6839dee..fd75bf7 100644 --- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py +++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py @@ -106,6 +106,39 @@ class TestAtlasServer(RMFTestCase): group='hadoop', mode=0644, ) + self.assertResourceCalled('Directory', '/usr/lib/ambari-logsearch-solr-client', + owner='solr', + group='solr', + create_parents = True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('Directory', '/var/log/ambari-logsearch-solr-client', + owner='solr', + group='solr', + create_parents=True, + cd_access='a', + mode=0755 + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh', + content=StaticFile('/usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh'), + owner='solr', + group='solr', + mode=0755, + ) + self.assertResourceCalled('File', '/usr/lib/ambari-logsearch-solr-client/log4j.properties', + content=InlineTemplate(self.getConfig()['configurations'][ + 'logsearch-solr-client-log4j']['content']), + owner='solr', + group='solr', + mode=0644, + ) + self.assertResourceCalled('File', '/var/log/ambari-logsearch-solr-client/solr-client.log', + owner='solr', + group='solr', + mode=0644, + content = '' + ) self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --download-config -d /tmp/solr_config_basic_configs_0.[0-9]* -cs basic_configs -rt 5 -i 10') self.assertResourceCalledRegexp('^Execute$', '^export JAVA_HOME=/usr/jdk64/jdk1.7.0_45 ; /usr/lib/ambari-logsearch-solr-client/solrCloudCli.sh -z c6401.ambari.apache.org/logsearch --upload-config -d /usr/lib/ambari-logsearch-solr/server/solr/configsets/basic_configs/conf -cs basic_configs -rt 5 -i 10') http://git-wip-us.apache.org/repos/asf/ambari/blob/c0f4d1b6/ambari-server/src/test/python/stacks/2.5/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json index 0b67405..1015593 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json @@ -172,7 +172,13 @@ "log.retention.hours": "168" }, "logsearch-solr-env": { - "logsearch_solr_znode": "/logsearch" + "logsearch_solr_znode": "/logsearch", + "logsearch_solr_user": "solr", + "logsearch_solr_group": "solr", + "logsearch_solr_client_log_dir" :"/var/log/ambari-logsearch-solr-client" + }, + "logsearch-solr-client-log4j" : { + "content" : "content" }, "application-properties": { "atlas.cluster.name" : "c2",
