Repository: ambari Updated Branches: refs/heads/branch-2.6 dc969b4f7 -> ab1d01bd1
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_server.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_server.py new file mode 100644 index 0000000..c4e99fa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_server.py @@ -0,0 +1,88 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import * +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.functions.security_commons import build_expectations, \ + cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ + FILE_TYPE_XML +from webhcat import webhcat +from webhcat_service import webhcat_service +from ambari_commons import OSConst +from ambari_commons.os_family_impl import OsFamilyImpl + + +class WebHCatServer(Script): + def install(self, env): + import params + self.install_packages(env) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) # FOR SECURITY + webhcat_service(action='start', upgrade_type=upgrade_type) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + webhcat_service(action='stop') + + def configure(self, env): + import params + env.set_params(params) + webhcat() + + +@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY) +class WebHCatServerWindows(WebHCatServer): + def status(self, env): + import status_params + env.set_params(status_params) + check_windows_service_status(status_params.webhcat_server_win_service_name) + + +@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) +class WebHCatServerDefault(WebHCatServer): + def status(self, env): + import status_params + env.set_params(status_params) + check_process_status(status_params.webhcat_pid_file) + + def pre_upgrade_restart(self, env, upgrade_type=None): + Logger.info("Executing WebHCat Stack Upgrade pre-restart") + import params + env.set_params(params) + + if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): + stack_select.select_packages(params.version) + + def get_log_folder(self): + import params + return params.hcat_log_dir + + def get_user(self): + import params + return params.webhcat_user + +if __name__ == "__main__": + WebHCatServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service.py new file mode 100644 index 0000000..ec8a0b7 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service.py @@ -0,0 +1,96 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Ambari Agent + +""" +from resource_management import * +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +from resource_management.core.logger import Logger +import traceback + + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def webhcat_service(action='start', rolling_restart=False): + import params + if action == 'start' or action == 'stop': + Service(params.webhcat_server_win_service_name, action=action) + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def webhcat_service(action='start', upgrade_type=None): + import params + + cmd = format('{webhcat_bin_dir}/webhcat_server.sh') + + if action == 'start': + daemon_cmd = format('cd {hcat_pid_dir} ; {cmd} start') + no_op_test = format('ls {webhcat_pid_file} >/dev/null 2>&1 && ps -p `cat {webhcat_pid_file}` >/dev/null 2>&1') + try: + Execute(daemon_cmd, + user=params.webhcat_user, + not_if=no_op_test) + except: + show_logs(params.hcat_log_dir, params.webhcat_user) + raise + elif action == 'stop': + try: + # try stopping WebHCat using its own script + graceful_stop(cmd) + except Fail: + show_logs(params.hcat_log_dir, params.webhcat_user) + Logger.info(traceback.format_exc()) + + # this will retrieve the PID + pid_expression = format("`cat {webhcat_pid_file}`") + + # the PID must exist AND'd with the process must be alive + # the return code here is going to be 0 IFF both conditions are met correctly + process_id_exists_command = format("ls {webhcat_pid_file} >/dev/null 2>&1 && ps -p {pid_expression} >/dev/null 2>&1") + + # kill command to run + daemon_hard_kill_cmd = format("{sudo} kill -9 {pid_expression}") + + # check to ensure that it has stopped by looking for the running PID and then killing + # it forcefully if it exists - the behavior of not-if/only-if is as follows: + # not_if return code IS 0 + # only_if return code is NOT 0 + Execute(daemon_hard_kill_cmd, + only_if = process_id_exists_command, + ignore_failures = True) + + try: + # check if stopped the process, else fail the task + Execute(format("! ({process_id_exists_command})")) + except: + show_logs(params.hcat_log_dir, params.webhcat_user) + raise + + File(params.webhcat_pid_file, action="delete") + +def graceful_stop(cmd): + """ + Attemps to stop WebHCat using its own shell script. On some versions this may not correctly + stop the daemon. + :param cmd: the command to run to stop the daemon + :return: + """ + import params + daemon_cmd = format('{cmd} stop') + + Execute(daemon_cmd, user = params.webhcat_user) http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service_check.py new file mode 100644 index 0000000..8e80d48 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/scripts/webhcat_service_check.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import urllib2 + +from resource_management import * +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from ambari_commons import OSConst +import time + +@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY) +def webhcat_service_check(): + Logger.info("Webhcat smoke test - service status") + + import params + # AMBARI-11633 [WinTP2] Webhcat service check fails + # Hive doesn't pass the environment variables correctly to child processes, which fails the smoke test. + # Reducing the amount of URLs checked to the minimum required. + #smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd") + #service = "WEBHCAT" + #Execute(format("cmd /C {smoke_cmd} {service}"), user=params.hcat_user, logoutput=True) + + url_tests = [ + "status", + #These are the failing ones: + #"ddl/database?user.name=hadoop", + #"ddl/database/default/table?user.name=hadoop" + ] + + + import socket + + url_host = socket.getfqdn() + url_port = params.config["configurations"]["webhcat-site"]["templeton.port"] + + for url_test in url_tests: + url_request = "http://{0}:{1}/templeton/v1/{2}".format(url_host, url_port, url_test) + url_response = None + + try: + # execute the query for the JSON that includes WebHCat status + url_response = urllib2.urlopen(url_request, timeout=30) + + status = url_response.getcode() + response = url_response.read() + + if status != 200: + Logger.warning("Webhcat service check status: {0}".format(status)) + Logger.info("Webhcat service check response: {0}".format(response)) + except urllib2.HTTPError as he: + raise Fail("Webhcat check {0} failed: {1}".format(url_request, he.msg)) + finally: + if url_response is not None: + try: + url_response.close() + except: + pass + + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def webhcat_service_check(): + import params + File(format("{tmp_dir}/templetonSmoke.sh"), + content= StaticFile('templetonSmoke.sh'), + mode=0755 + ) + + if params.security_enabled: + smokeuser_keytab=params.smoke_user_keytab + smoke_user_principal=params.smokeuser_principal + else: + smokeuser_keytab= "no_keytab" + smoke_user_principal="no_principal" + + unique_name = format("{smokeuser}.{timestamp}", timestamp = time.time()) + templeton_test_script = format("idtest.{unique_name}.pig") + templeton_test_input = format("/tmp/idtest.{unique_name}.in") + templeton_test_output = format("/tmp/idtest.{unique_name}.out") + + File(format("{tmp_dir}/{templeton_test_script}"), + content = Template("templeton_smoke.pig.j2", templeton_test_input=templeton_test_input, templeton_test_output=templeton_test_output), + owner=params.hdfs_user + ) + + params.HdfsResource(format("/tmp/{templeton_test_script}"), + action = "create_on_execute", + type = "file", + source = format("{tmp_dir}/{templeton_test_script}"), + owner = params.smokeuser + ) + + params.HdfsResource(templeton_test_input, + action = "create_on_execute", + type = "file", + source = "/etc/passwd", + owner = params.smokeuser + ) + + params.HdfsResource(None, action = "execute") + + cmd = format("{tmp_dir}/templetonSmoke.sh {webhcat_server_host[0]} {smokeuser} {templeton_port} {templeton_test_script} {smokeuser_keytab}" + " {security_param} {kinit_path_local} {smoke_user_principal}" + " {tmp_dir}") + + Execute(cmd, + tries=3, + try_sleep=5, + path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', + logoutput=True) + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hivemetastore.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hivemetastore.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hivemetastore.properties.j2 new file mode 100644 index 0000000..82f71c5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hivemetastore.properties.j2 @@ -0,0 +1,56 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_metric_collector %} + + *.period={{metrics_collection_period}} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink + *.sink.timeline.period={{metrics_collection_period}} + *.sink.timeline.sendInterval={{metrics_report_interval}}000 + *.sink.timeline.slave.host.name = {{hostname}} + + # HTTPS properties + *.sink.timeline.truststore.path = {{metric_truststore_path}} + *.sink.timeline.truststore.type = {{metric_truststore_type}} + *.sink.timeline.truststore.password = {{metric_truststore_password}} + + hivemetastore.sink.timeline.collector.hosts={{ams_collector_hosts}} + hivemetastore.sink.timeline.port={{metric_collector_port}} + hivemetastore.sink.timeline.protocol={{metric_collector_protocol}} + + +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hiveserver2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hiveserver2.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hiveserver2.properties.j2 new file mode 100644 index 0000000..f1bfa03 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-hiveserver2.properties.j2 @@ -0,0 +1,55 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_metric_collector %} + + *.period={{metrics_collection_period}} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink + *.sink.timeline.period={{metrics_collection_period}} + *.sink.timeline.sendInterval={{metrics_report_interval}}000 + *.sink.timeline.slave.host.name = {{hostname}} + + # HTTPS properties + *.sink.timeline.truststore.path = {{metric_truststore_path}} + *.sink.timeline.truststore.type = {{metric_truststore_type}} + *.sink.timeline.truststore.password = {{metric_truststore_password}} + + hiveserver2.sink.timeline.collector.hosts={{ams_collector_hosts}} + hiveserver2.sink.timeline.port={{metric_collector_port}} + hiveserver2.sink.timeline.protocol={{metric_collector_protocol}} + +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llapdaemon.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llapdaemon.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llapdaemon.j2 new file mode 100644 index 0000000..2a435c1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llapdaemon.j2 @@ -0,0 +1,54 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_metric_collector %} + + *.period={{metrics_collection_period}} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink + *.sink.timeline.period={{metrics_collection_period}} + *.sink.timeline.sendInterval={{metrics_report_interval}}000 + + # HTTPS properties + *.sink.timeline.truststore.path = {{metric_truststore_path}} + *.sink.timeline.truststore.type = {{metric_truststore_type}} + *.sink.timeline.truststore.password = {{metric_truststore_password}} + + llapdaemon.sink.timeline.collector.hosts={{ams_collector_hosts}} + llapdaemon.sink.timeline.port={{metric_collector_port}} + llapdaemon.sink.timeline.protocol={{metric_collector_protocol}} + +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llaptaskscheduler.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llaptaskscheduler.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llaptaskscheduler.j2 new file mode 100644 index 0000000..7b94c69 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hadoop-metrics2-llaptaskscheduler.j2 @@ -0,0 +1,54 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +{% if has_metric_collector %} + + *.period={{metrics_collection_period}} + *.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar + *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink + *.sink.timeline.period={{metrics_collection_period}} + *.sink.timeline.sendInterval={{metrics_report_interval}}000 + + # HTTPS properties + *.sink.timeline.truststore.path = {{metric_truststore_path}} + *.sink.timeline.truststore.type = {{metric_truststore_type}} + *.sink.timeline.truststore.password = {{metric_truststore_password}} + + llaptaskscheduler.sink.timeline.collector.hosts={{ams_collector_hosts}} + llaptaskscheduler.sink.timeline.port={{metric_collector_port}} + llaptaskscheduler.sink.timeline.protocol={{metric_collector_protocol}} + +{% endif %} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hive.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hive.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hive.conf.j2 new file mode 100644 index 0000000..5af53d0 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/hive.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{hive_user}} - nofile {{hive_user_nofile_limit}} +{{hive_user}} - nproc {{hive_user_nproc_limit}} http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2.sh.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2.sh.j2 new file mode 100644 index 0000000..70b418c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2.sh.j2 @@ -0,0 +1,24 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +HIVE_SERVER2_OPTS=" -hiveconf hive.log.file=hiveserver2.log -hiveconf hive.log.dir=$5" +HIVE_CONF_DIR=$4 {{hive_bin}}/hiveserver2 -hiveconf hive.metastore.uris=" " ${HIVE_SERVER2_OPTS} > $1 2> $2 & +echo $!|cat>$3 http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2Interactive.sh.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2Interactive.sh.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2Interactive.sh.j2 new file mode 100644 index 0000000..6062a7e --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/startHiveserver2Interactive.sh.j2 @@ -0,0 +1,24 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +HIVE_SERVER2_INTERACTIVE_OPTS=" -hiveconf hive.log.file=hiveserver2Interactive.log -hiveconf hive.log.dir=$5" +HIVE_INTERACTIVE_CONF_DIR=$4 {{hive_interactive_bin}}/hiveserver2 -hiveconf hive.metastore.uris=" " ${HIVE_SERVER2_INTERACTIVE_OPTS} > $1 2> $2 & +echo $!|cat>$3 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/templeton_smoke.pig.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/templeton_smoke.pig.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/templeton_smoke.pig.j2 new file mode 100644 index 0000000..3153e81 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/templeton_smoke.pig.j2 @@ -0,0 +1,24 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# + +A = load '{{templeton_test_input}}' using PigStorage(':'); +B = foreach A generate \$0 as id; +store B into '{{templeton_test_output}}'; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/zkmigrator_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/zkmigrator_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/zkmigrator_jaas.conf.j2 new file mode 100644 index 0000000..e7adfd3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/HIVE/package/templates/zkmigrator_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{hive_keytab}}" + principal="{{hive_principal}}"; +}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py index 0575805..7589cea 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/services/OOZIE/package/scripts/oozie.py @@ -306,11 +306,12 @@ def oozie_server_specific(upgrade_type): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) - if params.lzo_enabled: - install_lzo_if_needed() - Execute(format('{sudo} cp /usr/lib/hadoop-lzo/lib/hadoop-lzo*.jar {oozie_lib_dir}'), - not_if = no_op_test, - ) + # LZO if needed should be installed manually + # if params.lzo_enabled: + # install_lzo_if_needed() + # Execute(format('{sudo} cp /usr/lib/hadoop-lzo/lib/hadoop-lzo*.jar {oozie_lib_dir}'), + # not_if = no_op_test, + # ) prepare_war(params) http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py index 9334de0..9b36173 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py @@ -115,7 +115,8 @@ def hdfs(name=None): content=Template("slaves.j2") ) - install_lzo_if_needed() + # LZO if needed should be installed manually + # install_lzo_if_needed() def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py index f451a65..5847a6e 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py @@ -49,7 +49,8 @@ from ambari_commons import OSConst def hive(name=None): import params - install_lzo_if_needed() + # LZO if needed should be installed manually + # install_lzo_if_needed() XmlConfig("hive-site.xml", conf_dir = params.hive_conf_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py index 0459432..5d4bddb 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py @@ -100,12 +100,13 @@ def oozie(is_server=False): pass oozie_ownership() - - if params.lzo_enabled: - install_lzo_if_needed() - Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - not_if = no_op_test, - ) + + # LZO if needed should be installed manually + # if params.lzo_enabled: + # install_lzo_if_needed() + # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + # not_if = no_op_test, + # ) if is_server: oozie_server_specific() http://git-wip-us.apache.org/repos/asf/ambari/blob/ab1d01bd/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py index 9006d2e..c528338 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py @@ -51,8 +51,9 @@ def create_local_dir(dir_name): def yarn(name = None): import params - - install_lzo_if_needed() + + # LZO if needed should be installed manually + # install_lzo_if_needed() if name == "historyserver": if params.yarn_log_aggregation_enabled:
