Repository: ambari Updated Branches: refs/heads/branch-1.7.0 6f5db3166 -> 7d0064d7c
AMBARI-7551 Create mahout definition for bigtop (rmeneses via jaoki) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7d0064d7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7d0064d7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7d0064d7 Branch: refs/heads/branch-1.7.0 Commit: 7d0064d7cf9b2489f81b88ab7333cd9d421c89b3 Parents: 6f5db31 Author: Jun Aoki <ja...@apache.org> Authored: Wed Oct 8 11:41:12 2014 -0700 Committer: Jun Aoki <ja...@apache.org> Committed: Wed Oct 8 11:41:12 2014 -0700 ---------------------------------------------------------------------- .../stacks/BIGTOP/0.8/role_command_order.json | 2 + .../BIGTOP/0.8/services/MAHOUT/metainfo.xml | 66 ++++++++++++++ .../services/MAHOUT/package/scripts/mahout.py | 66 ++++++++++++++ .../MAHOUT/package/scripts/mahout_client.py | 36 ++++++++ .../services/MAHOUT/package/scripts/params.py | 55 ++++++++++++ .../MAHOUT/package/scripts/service_check.py | 92 ++++++++++++++++++++ .../MAHOUT/package/templates/mahout-env.sh.j2 | 34 ++++++++ 7 files changed, 351 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json index 2604b6d..69fcdac 100644 --- a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/role_command_order.json @@ -26,6 +26,7 @@ "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START"], "HCAT_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START"], "PIG_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], + "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "ZOOKEEPER_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], "ZOOKEEPER_QUORUM_SERVICE_CHECK-SERVICE_CHECK": ["ZOOKEEPER_SERVER-START"], @@ -55,6 +56,7 @@ "YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"], "RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"], "PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], + "MAHOUT_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"], "NAMENODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"], "DATANODE-STOP": ["RESOURCEMANAGER-STOP", "NODEMANAGER-STOP", http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml new file mode 100644 index 0000000..058f047 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/metainfo.xml @@ -0,0 +1,66 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>MAHOUT</name> + <displayName>Mahout</displayName> + <comment>The Apache Mahout project's goal is to build a scalable machine learning library</comment> + <version>0.9.666</version> + <components> + <component> + <name>MAHOUT</name> + <displayName>Mahout Client</displayName> + <category>CLIENT</category> + <cardinality>0+</cardinality> + <commandScript> + <script>scripts/mahout_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>mahout</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + </requiredServices> + + <configuration-dependencies> + <config-type>global</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py new file mode 100644 index 0000000..3984c89 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout.py @@ -0,0 +1,66 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os + +from resource_management import * + +def mahout(): + import params + + Directory( params.mahout_conf_dir, + owner = params.hdfs_user, + group = params.user_group + ) + + mahout_TemplateConfig( ['mahout-env.sh']) + + # mahout_properties is always set to a default even if it's not in the payload + File(format("{mahout_conf_dir}/mahout.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.mahout_properties + ) + + if params.log4j_props: + File(format("{mahout_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{mahout_conf_dir}/log4j.properties"))): + File(format("{mahout_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user + ) + +def mahout_TemplateConfig(name): + import params + + if not isinstance(name, list): + name = [name] + + for x in name: + TemplateConfig( format("{mahout_conf_dir}/{x}"), + owner = params.hdfs_user + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py new file mode 100644 index 0000000..8457bfc --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/mahout_client.py @@ -0,0 +1,36 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Ambari Agent +""" +import sys +from resource_management import * +from mahout import mahout + +class MahoutClient(Script): + + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + mahout() + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + MahoutClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py new file mode 100644 index 0000000..d6cd33f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/params.py @@ -0,0 +1,55 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management import * + +# server configurations +config = Script.get_config() + +hadoop_log_dir = "/var/log/hadoop" +mahout_conf_dir = "/etc/mahout/conf" +hadoop_conf_dir = "/etc/hadoop/conf" +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] + +smokeuser = config['configurations']['cluster-env']['smokeuser'] +user_group = config['configurations']['cluster-env']['user_group'] +_authentication = config['configurations']['core-site']['hadoop.security.authentication'] +security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] +hadoop_home = "/usr/lib/hadoop/" + +# mahout.properties - if not in the JSON command, then we need to esnure some +# basic properties are set; this is a safety mechanism +if (('mahout-properties' in config['configurations']) and ('mahout-content' in config['configurations']['mahout-properties'])): + mahout_properties = config['configurations']['mahout-properties']['mahout-content'] +else: + mahout_properties = "mahout.location.check.strict=false" + +# log4j.properties +if (('mahout-log4j' in config['configurations']) and ('content' in config['configurations']['mahout-log4j'])): + log4j_props = config['configurations']['mahout-log4j']['content'] +else: + log4j_props = None http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py new file mode 100644 index 0000000..bb2ec4d --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/scripts/service_check.py @@ -0,0 +1,92 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os +from resource_management import * + +class MahoutServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + # prepare the input file content + input_file = os.path.join("/tmp", "mahout_input.csv") + input_file_content = """\ +1,101,5.0 +1,102,5.0 +1,103,2.5 +2,101,2.0 +2,102,2.5 +2,103,5.0 +2,104,2.0 +3,101,2.5 +3,104,4.0 +3,105,4.5 +3,107,5.0""" + + File(input_file, content=input_file_content, mode=644) + + # create the log dir for the smoke user + Directory(os.path.join(params.hadoop_log_dir, params.smokeuser), + owner = params.smokeuser, + group = params.user_group, + mode = 755) + + # transfer the input file to hdfs + recommenderdata_dir = "recommenderdata" + recommenderoutput_dir = "recommenderoutput" + cleanup_cmd = format("fs -rm -r {recommenderdata_dir} {recommenderoutput_dir} temp") + #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command + create_file_cmd = format("{cleanup_cmd}; hadoop fs -put {input_file} {recommenderdata_dir}") #TODO: inconsistent that second command needs hadoop + + test_cmd_cat = "mahout cat /etc/passwd" + test_cmd_recommendation = format("mahout recommenditembased --input {recommenderdata_dir} --output {recommenderoutput_dir} -s SIMILARITY_COOCCURRENCE") + + ExecuteHadoop(create_file_cmd, + tries = 3, + try_sleep = 5, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + # for kinit run + keytab = params.smoke_user_keytab, + security_enabled = params.security_enabled, + kinit_path_local = params.kinit_path_local, + logoutput = True + ) + + Execute(test_cmd_cat, + tries = 3, + try_sleep = 5, + path = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user = params.smokeuser, + logoutput = True + ) + + Execute(test_cmd_recommendation, + tries = 3, + try_sleep = 5, + path = '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + user = params.smokeuser, + logoutput = True, + ) + +if __name__ == "__main__": + MahoutServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/7d0064d7/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2 new file mode 100644 index 0000000..3f97f10 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/MAHOUT/package/templates/mahout-env.sh.j2 @@ -0,0 +1,34 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +JAVA_HOME={{java64_home}} +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
