AMBARI-20652. HDP 3.0 TP - add Pig client: configs, metainfo, descriptors, service advisor and etc.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9fec5ac5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9fec5ac5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9fec5ac5 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 9fec5ac5ae9208f18d9e4180d8482dc14f05c8b3 Parents: 0ffb42b Author: Vitaly Brodetskyi <[email protected]> Authored: Mon Apr 3 15:37:51 2017 +0300 Committer: Vitaly Brodetskyi <[email protected]> Committed: Mon Apr 3 15:38:44 2017 +0300 ---------------------------------------------------------------------- .../PIG/0.16.1.3.0/configuration/pig-env.xml | 41 +++++ .../PIG/0.16.1.3.0/configuration/pig-log4j.xml | 65 +++++++ .../0.16.1.3.0/configuration/pig-properties.xml | 93 ++++++++++ .../PIG/0.16.1.3.0/kerberos.json | 17 ++ .../common-services/PIG/0.16.1.3.0/metainfo.xml | 101 +++++++++++ .../PIG/0.16.1.3.0/package/files/pigSmoke.sh | 18 ++ .../PIG/0.16.1.3.0/package/scripts/params.py | 31 ++++ .../0.16.1.3.0/package/scripts/params_linux.py | 104 +++++++++++ .../package/scripts/params_windows.py | 45 +++++ .../PIG/0.16.1.3.0/package/scripts/pig.py | 80 +++++++++ .../0.16.1.3.0/package/scripts/pig_client.py | 71 ++++++++ .../0.16.1.3.0/package/scripts/service_check.py | 142 +++++++++++++++ .../PIG/0.16.1.3.0/service_advisor.py | 171 +++++++++++++++++++ .../stacks/HDP/3.0/services/PIG/metainfo.xml | 27 +++ 14 files changed, 1006 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml new file mode 100644 index 0000000..11afa11 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_adding_forbidden="true"> + <!-- pig-env.sh --> + <property> + <name>content</name> + <display-name>pig-env template</display-name> + <description>This is the jinja template for pig-env.sh file</description> + <value> +JAVA_HOME={{java64_home}} +HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}} + +if [ -d "/usr/lib/tez" ]; then + PIG_OPTS="$PIG_OPTS -Dmapreduce.framework.name=yarn" +fi + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="fallse"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml new file mode 100644 index 0000000..e5ff9b6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>pig-log4j template</display-name> + <description>Custom log4j.properties</description> + <value> +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# + +# ***** Set root logger level to DEBUG and its only appender to A. +log4j.logger.org.apache.pig=info, A + +# ***** A is set to be a ConsoleAppender. +log4j.appender.A=org.apache.log4j.ConsoleAppender +# ***** A uses PatternLayout. +log4j.appender.A.layout=org.apache.log4j.PatternLayout +log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml new file mode 100644 index 0000000..fb43f6e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml @@ -0,0 +1,93 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <display-name>pig-properties template</display-name> + <description>Describe all the Pig agent configurations</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments. +# see bin/pig -help + +# brief logging (no timestamps) +brief=false + +# debug level, INFO is default +debug=INFO + +# verbose print all log messages to screen (default to print only INFO and above to screen) +verbose=false + +# exectype local|mapreduce|tez, mapreduce is default +exectype=tez + +# Enable insertion of information about script into hadoop job conf +pig.script.info.enabled=true + +# Do not spill temp files smaller than this size (bytes) +pig.spill.size.threshold=5000000 + +# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes) +# This should help reduce the number of files being spilled. +pig.spill.gc.activation.size=40000000 + +# the following two parameters are to help estimate the reducer number +pig.exec.reducers.bytes.per.reducer=1000000000 +pig.exec.reducers.max=999 + +# Temporary location to store the intermediate data. +pig.temp.dir=/tmp/ + +# Threshold for merging FRJoin fragment files +pig.files.concatenation.threshold=100 +pig.optimistic.files.concatenation=false; + +pig.disable.counter=false + +hcat.bin=/usr/bin/hcat + + </value> + <value-attributes> + <type>content</type> + <empty-value-valid>true</empty-value-valid> + <show-property-name>false</show-property-name> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json new file mode 100644 index 0000000..22dd6cb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json @@ -0,0 +1,17 @@ +{ + "services": [ + { + "name": "PIG", + "components": [ + { + "name": "PIG", + "identities": [ + { + "name": "/HDFS/NAMENODE/hdfs" + } + ] + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml new file mode 100644 index 0000000..041255a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml @@ -0,0 +1,101 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>PIG</name> + <displayName>Pig</displayName> + <comment>Scripting platform for analyzing large datasets</comment> + <version>0.16.1.3.0</version> + <components> + <component> + <name>PIG</name> + <displayName>Pig Client</displayName> + <category>CLIENT</category> + <cardinality>0+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/pig_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>pig-env.sh</fileName> + <dictionaryName>pig-env</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>log4j.properties</fileName> + <dictionaryName>pig-log4j</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>pig.properties</fileName> + <dictionaryName>pig-properties</dictionaryName> + </configFile> + </configFiles> + </component> + </components> + <osSpecifics> + <osSpecific> + <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily> + <packages> + <package> + <name>pig_${stack_version}</name> + </package> + <package> + <name>datafu_${stack_version}</name> + </package> + </packages> + </osSpecific> + <osSpecific> + <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily> + <packages> + <package> + <name>pig-${stack_version}</name> + </package> + <package> + <name>datafu-${stack_version}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>YARN</service> + <service>TEZ</service> + </requiredServices> + + <configuration-dependencies> + <config-type>pig-env</config-type> + <config-type>pig-log4j</config-type> + <config-type>pig-properties</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh new file mode 100644 index 0000000..a22456e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh @@ -0,0 +1,18 @@ +/*Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License */ + +A = load 'passwd' using PigStorage(':'); +B = foreach A generate \$0 as id; +store B into 'pigsmoke.out'; http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py new file mode 100644 index 0000000..3aebda0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py @@ -0,0 +1,31 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from ambari_commons import OSCheck +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs() + http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py new file mode 100644 index 0000000..5af1274 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py @@ -0,0 +1,104 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +from resource_management.libraries.script.script import Script +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources + +# server configurations +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +stack_name = default("/hostLevelParams/stack_name", None) +stack_root = Script.get_stack_root() + +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +# hadoop default parameters +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") +pig_conf_dir = "/etc/pig/conf" +hadoop_home = '/usr' +pig_bin_dir = "" + +# hadoop parameters for stack versions supporting rolling_upgrade +if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted): + pig_conf_dir = format("{stack_root}/current/pig-client/conf") + hadoop_home = stack_select.get_hadoop_dir("home") + pig_bin_dir = format("{stack_root}/current/pig-client/bin") + +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_hdfs_user_mode = 0770 +user_group = config['configurations']['cluster-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +pig_env_sh_template = config['configurations']['pig-env']['content'] + +# not supporting 32 bit jdk. +java64_home = config['hostLevelParams']['java_home'] + +pig_properties = config['configurations']['pig-properties']['content'] + +log4j_props = config['configurations']['pig-log4j']['content'] + + + +hdfs_site = config['configurations']['hdfs-site'] +default_fs = config['configurations']['core-site']['fs.defaultFS'] + +dfs_type = default("/commandParams/dfs_type", "") + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create hdfs directory we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources(), + dfs_type = dfs_type + ) + http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py new file mode 100644 index 0000000..bd9654f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py @@ -0,0 +1,45 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management.libraries.script.script import Script + +# server configurations +config = Script.get_config() + +stack_root = None +pig_home = None +pig_conf_dir = None +try: + stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],"..")) + pig_home = os.environ['PIG_HOME'] + pig_conf_dir = os.path.join(pig_home,'conf') +except: + pass + +pig_properties = config['configurations']['pig-properties']['content'] + +if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])): + log4j_props = config['configurations']['pig-log4j']['content'] +else: + log4j_props = None + +hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] +pig_user = hadoop_user +hdfs_user = hadoop_user http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py new file mode 100644 index 0000000..b6825d5 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py @@ -0,0 +1,80 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +import os +from resource_management.core.resources.system import Directory, File +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions.format import format +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def pig(): + import params + + Directory( params.pig_conf_dir, + create_parents = True, + owner = params.hdfs_user, + group = params.user_group + ) + + File(format("{pig_conf_dir}/pig-env.sh"), + owner=params.hdfs_user, + mode=0755, + content=InlineTemplate(params.pig_env_sh_template) + ) + + # pig_properties is always set to a default even if it's not in the payload + File(format("{params.pig_conf_dir}/pig.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.pig_properties + ) + + if (params.log4j_props != None): + File(format("{params.pig_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))): + File(format("{params.pig_conf_dir}/log4j.properties"), + mode=0644, + group=params.user_group, + owner=params.hdfs_user + ) + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def pig(): + import params + File(os.path.join(params.pig_conf_dir, "pig.properties"), + mode="f", + owner=params.pig_user, + content=params.pig_properties + ) + + if (params.log4j_props != None): + File(os.path.join(params.pig_conf_dir, "log4j.properties"), + mode='f', + owner=params.pig_user, + content=params.log4j_props + ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py new file mode 100644 index 0000000..6fdcef8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py @@ -0,0 +1,71 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import sys +import os +from resource_management.libraries.script.script import Script +from resource_management.libraries.functions import conf_select, stack_select +from resource_management.libraries.functions.constants import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from pig import pig +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.libraries.functions.default import default +from resource_management.core.exceptions import ClientComponentHasNoStatus + +class PigClient(Script): + def configure(self, env): + import params + env.set_params(params) + pig() + + def status(self, env): + raise ClientComponentHasNoStatus() + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class PigClientLinux(PigClient): + def get_component_name(self): + return "hadoop-client" + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + conf_select.select(params.stack_name, "pig", params.version) + conf_select.select(params.stack_name, "hadoop", params.version) + stack_select.select("hadoop-client", params.version) # includes pig-client + + def install(self, env): + self.install_packages(env) + self.configure(env) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class PigClientWindows(PigClient): + + def install(self, env): + import params + if params.pig_home is None: + self.install_packages(env) + self.configure(env) + +if __name__ == "__main__": + PigClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py new file mode 100644 index 0000000..ba4082e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py @@ -0,0 +1,142 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" + +import os + +from resource_management.core.resources.system import Execute, File +from resource_management.core.source import InlineTemplate, StaticFile +from resource_management.libraries.functions.copy_tarball import copy_to_hdfs +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.script.script import Script +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +class PigServiceCheck(Script): + pass + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class PigServiceCheckLinux(PigServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + + input_file = format('/user/{smokeuser}/passwd') + output_dir = format('/user/{smokeuser}/pigsmoke.out') + + params.HdfsResource(format("/user/{smokeuser}"), + type="directory", + action="create_on_execute", + owner=params.smokeuser, + mode=params.smoke_hdfs_user_mode, + ) + + params.HdfsResource(output_dir, + type="directory", + action="delete_on_execute", + owner=params.smokeuser, + ) + params.HdfsResource(input_file, + type="file", + source="/etc/passwd", + action="create_on_execute", + owner=params.smokeuser, + ) + params.HdfsResource(None, action="execute") + + if params.security_enabled: + kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};") + Execute(kinit_cmd, + user=params.smokeuser + ) + + File( format("{tmp_dir}/pigSmoke.sh"), + content = StaticFile("pigSmoke.sh"), + mode = 0755 + ) + + # check for Pig-on-M/R + Execute( format("pig {tmp_dir}/pigSmoke.sh"), + tries = 3, + try_sleep = 5, + path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), + user = params.smokeuser, + logoutput = True + ) + + test_cmd = format("fs -test -e {output_dir}") + ExecuteHadoop( test_cmd, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + bin_dir = params.hadoop_bin_dir + ) + + if params.stack_version_formatted and check_stack_feature(StackFeature.PIG_ON_TEZ, params.stack_version_formatted): + # cleanup results from previous test + params.HdfsResource(output_dir, + type="directory", + action="delete_on_execute", + owner=params.smokeuser, + ) + params.HdfsResource(input_file, + type="file", + source="/etc/passwd", + action="create_on_execute", + owner=params.smokeuser, + ) + + # Check for Pig-on-Tez + resource_created = copy_to_hdfs( + "tez", params.user_group, + params.hdfs_user, + skip=params.sysprep_skip_copy_tarballs_hdfs) + if resource_created: + params.HdfsResource(None, action="execute") + + Execute(format("pig -x tez {tmp_dir}/pigSmoke.sh"), + tries = 3, + try_sleep = 5, + path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'), + user = params.smokeuser, + logoutput = True + ) + + ExecuteHadoop(test_cmd, + user = params.smokeuser, + conf_dir = params.hadoop_conf_dir, + bin_dir = params.hadoop_bin_dir + ) + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class PigServiceCheckWindows(PigServiceCheck): + def service_check(self, env): + import params + env.set_params(params) + smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd") + service = "PIG" + Execute(format("cmd /C {smoke_cmd} {service}", smoke_cmd=smoke_cmd, service=service), logoutput=True, user=params.pig_user, timeout=300) + +if __name__ == "__main__": + PigServiceCheck().execute() + http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py new file mode 100644 index 0000000..8f89c5d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py @@ -0,0 +1,171 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback +import re +import socket +import fnmatch + + +from resource_management.core.logger import Logger + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class PigServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(PigServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return [] + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + """ + Entry point. + Must be overriden in child class. + """ + #Logger.info("Class: %s, Method: %s. Recommending Service Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + recommender = PigRecommender() + + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + validator = PigValidator() + # Calls the methods of the validator using arguments, + # method(siteProperties, siteRecommendations, configurations, services, hosts) + return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults, services, hosts, validator.validators) + + + +class PigRecommender(service_advisor.ServiceAdvisor): + """ + Pig Recommender suggests properties when adding the service for the first time or modifying configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(PigRecommender, self) + self.as_super.__init__(*args, **kwargs) + + + + + + + +class PigValidator(service_advisor.ServiceAdvisor): + """ + Pig Validator checks the correctness of properties whenever the service is first added or the user attempts to + change configs via the UI. + """ + + def __init__(self, *args, **kwargs): + self.as_super = super(PigValidator, self) + self.as_super.__init__(*args, **kwargs) + + self.validators = [] + + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9fec5ac5/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml new file mode 100644 index 0000000..fd664d0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml @@ -0,0 +1,27 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>PIG</name> + <version>0.16.1.3.0</version> + <extends>common-services/PIG/0.16.1.3.0</extends> + </service> + </services> +</metainfo>
