http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py new file mode 100755 index 0000000..d446baa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/scripts/status_params.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from ambari_commons import OSCheck + +if OSCheck.is_windows_family(): + from params_windows import * +else: + from params_linux import * + +hbase_pid_dir = config['configurations']['ams-hbase-env']['hbase_pid_dir'] +hbase_user = ams_user +ams_collector_pid_dir = config['configurations']['ams-env']['metrics_collector_pid_dir'] +ams_monitor_pid_dir = config['configurations']['ams-env']['metrics_monitor_pid_dir'] + +security_enabled = config['configurations']['cluster-env']['security_enabled'] +ams_hbase_conf_dir = format("{hbase_conf_dir}") + +kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +hostname = config['hostname'] +tmp_dir = Script.get_tmp_dir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2 new file mode 100755 index 0000000..c5fbc9b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams.conf.j2 @@ -0,0 +1,35 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{ams_user}} - nofile {{max_open_files_limit}} +{{ams_user}} - nproc 65536 http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2 new file mode 100755 index 0000000..f7f00eb --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_collector_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{ams_collector_keytab_path}}" +principal="{{ams_collector_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2 new file mode 100755 index 0000000..1929548 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/ams_zookeeper_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Server { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{ams_zookeeper_keytab}}" +principal="{{ams_zookeeper_principal_name}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2 new file mode 100755 index 0000000..a65ea88 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hadoop-metrics2-hbase.properties.j2 @@ -0,0 +1,63 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink|jmx].[instance].[options] +# See package.html for org.apache.hadoop.metrics2 for details + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.period=30 +hbase.collector={{metric_collector_host}}:{{metric_collector_port}} + +jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +jvm.period=30 +jvm.collector={{metric_collector_host}}:{{metric_collector_port}} + +rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +rpc.period=30 +rpc.collector={{metric_collector_host}}:{{metric_collector_port}} + +*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar +*.sink.timeline.slave.host.name={{hostname}} +hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +hbase.sink.timeline.period={{metrics_collection_period}} +hbase.sink.timeline.sendInterval={{metrics_report_interval}}000 +hbase.sink.timeline.collector={{metric_collector_host}}:{{metric_collector_port}} +hbase.sink.timeline.serviceName-prefix=ams + +# Switch off metrics generation on a per region basis +*.source.filter.class=org.apache.hadoop.metrics2.filter.GlobFilter +hbase.*.source.filter.exclude=*Regions* http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2 new file mode 100755 index 0000000..38f9721 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_client_jaas.conf.j2 @@ -0,0 +1,23 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=false +useTicketCache=true; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2 new file mode 100755 index 0000000..c29c674 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_grant_permissions.j2 @@ -0,0 +1,39 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +grant '{{smoke_test_user}}', '{{smokeuser_permissions}}' +exit http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2 new file mode 100755 index 0000000..a93c36c --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_master_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{master_keytab_path}}" +principal="{{master_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2 new file mode 100755 index 0000000..7097481 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/hbase_regionserver_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { +com.sun.security.auth.module.Krb5LoginModule required +useKeyTab=true +storeKey=true +useTicketCache=false +keyTab="{{regionserver_keytab_path}}" +principal="{{regionserver_jaas_princ}}"; +}; http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2 new file mode 100755 index 0000000..aa03d195 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_groups.conf.j2 @@ -0,0 +1,37 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{ + "host_metric_groups": { + "all": { + "collect_every": "10", + "metrics": [ + { + "name": "bytes_out", + "value_threshold": "128" + } + ] + } + }, + "process_metric_groups": { + "": { + "collect_every": "15", + "metrics": [] + } + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2 new file mode 100755 index 0000000..fc86a58 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/metric_monitor.ini.j2 @@ -0,0 +1,31 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +[default] +debug_level = INFO +metrics_server = {{metric_collector_host}}:{{metric_collector_port}} +hostname = {{hostname}} +enable_time_threshold = false +enable_value_threshold = false + +[emitter] +send_interval = {{metrics_report_interval}} + +[collector] +collector_sleep_interval = 5 +max_queue_size = 5000 http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2 new file mode 100755 index 0000000..81d060b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/regionservers.j2 @@ -0,0 +1,20 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% for host in rs_hosts %}{{host}} +{% endfor %} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2 new file mode 100755 index 0000000..2ee0efa --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/AMBARI_METRICS/package/templates/smoketest_metrics.json.j2 @@ -0,0 +1,15 @@ +{ + "metrics": [ + { + "metricname": "AMBARI_METRICS.SmokeTest.FakeMetric", + "appid": "amssmoketestfake", + "hostname": "{{hostname}}", + "timestamp": {{current_time}}, + "starttime": {{current_time}}, + "metrics": { + "{{current_time}}": {{random1}}, + "{{current_time + 1000}}": {{current_time}} + } + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json new file mode 100755 index 0000000..4603d22 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/alerts.json @@ -0,0 +1,27 @@ +{ + "FLUME": { + "service": [], + "FLUME_HANDLER": [ + { + "name": "flume_agent_status", + "label": "Flume Agent Status", + "description": "This host-level alert is triggered if any of the expected flume agent processes are not available.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "FLUME/1.5.2.4.1/package/alerts/alert_flume_agent_status.py", + "parameters": [ + { + "name": "run.directory", + "display_name": "Run Directory", + "value": "/var/run/flume", + "type": "STRING", + "description": "The directory where flume agent processes will place their PID files." + } + ] + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml new file mode 100755 index 0000000..582e181 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-conf.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false"> + <property> + <name>content</name> + <description>Flume agent configurations. + Specifying the configuration here applies to all hosts in this configuration-group. Please use host specific configuration-groups to provide different configurations on different hosts. + For configuring multiple agents on the same host, provide the combined agent configurations here. Each agent should have a different name.</description> + <value> +# Flume agent config + </value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <show-property-name>false</show-property-name> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml new file mode 100755 index 0000000..a448a66 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/configuration/flume-env.xml @@ -0,0 +1,90 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>flume_conf_dir</name> + <display-name>Flume Conf Dir</display-name> + <value>/etc/flume/conf</value> + <description>Location to save configuration files</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + </property> + <property> + <name>flume_log_dir</name> + <display-name>Flume Log Dir</display-name> + <value>/var/log/flume</value> + <description>Location to save log files</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + <editable-only-at-install>true</editable-only-at-install> + </value-attributes> + </property> + <property> + <name>flume_user</name> + <value>flume</value> + <property-type>USER</property-type> + <description>Flume User</description> + </property> + + <!-- flume-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for flume-env.sh file</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced +# during Flume startup. + +# Enviroment variables can be set here. + +export JAVA_HOME={{java_home}} + +# Give Flume more memory and pre-allocate, enable remote monitoring via JMX +# export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote" + +# Note that the Flume conf directory is always included in the classpath. +#FLUME_CLASSPATH="" + +export HIVE_HOME={{flume_hive_home}} +export HCAT_HOME={{flume_hcat_home}} + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml new file mode 100755 index 0000000..a19eff1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metainfo.xml @@ -0,0 +1,69 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>FLUME</name> + <displayName>Flume</displayName> + <comment>A distributed service for collecting, aggregating, and moving large amounts of streaming data into HDFS</comment> + <version>1.5.2</version> + <components> + <component> + <name>FLUME_HANDLER</name> + <displayName>Flume</displayName> + <category>SLAVE</category> + <cardinality>1+</cardinality> + <versionAdvertised>true</versionAdvertised> + <commandScript> + <script>scripts/flume_handler.py</script> + <scriptType>PYTHON</scriptType> + <timeout>1200</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>flume</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/flume_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>HDFS</service> + </requiredServices> + + <configuration-dependencies> + <config-type>flume-env</config-type> + <config-type>flume-conf</config-type> + </configuration-dependencies> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json new file mode 100755 index 0000000..b0990ba --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/metrics.json @@ -0,0 +1,430 @@ +{ + "FLUME_HANDLER": { + "Component": [ + { + "type": "ganglia", + "metrics": { + "default": { + "metrics/flume/flume/CHANNEL/ChannelCapacity": { + "metric": "ChannelCapacity", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeAttemptCount": { + "metric": "EventTakeAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount": { + "metric": "EventTakeSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutAttemptCount": { + "metric": "EventPutAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelFillPercentage": { + "metric": "ChannelFillPercentage", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize": { + "metric": "ChannelSize", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount": { + "metric": "EventPutSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionCreatedCount": { + "metric": "ConnectionCreatedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchCompleteCount": { + "metric": "BatchCompleteCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/EventDrainSuccessCount": { + "metric": "EventDrainSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/EventDrainAttemptCount": { + "metric": "EventDrainAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionFailedCount": { + "metric": "ConnectionFailedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchUnderflowCount": { + "metric": "BatchUnderflowCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionClosedCount": { + "metric": "ConnectionClosedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchEmptyCount": { + "metric": "BatchEmptyCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendBatchReceivedCount": { + "metric": "AppendBatchReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendAcceptedCount": { + "metric": "AppendAcceptedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/OpenConnectionCount": { + "metric": "OpenConnectionCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendBatchAcceptedCount": { + "metric": "AppendBatchAcceptedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendReceivedCount": { + "metric": "AppendReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/EventReceivedCount": { + "metric": "EventReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/EventAcceptedCount": { + "metric": "EventAcceptedCount", + "pointInTime": true, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/min": { + "metric": "EventTakeSuccessCount._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/max": { + "metric": "EventTakeSuccessCount._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/avg": { + "metric": "EventTakeSuccessCount._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/sum": { + "metric": "EventTakeSuccessCount._rate._sum", + "pointInTime": false, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/avg": { + "metric": "EventPutSuccessCount._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/max": { + "metric": "EventPutSuccessCount._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/min": { + "metric": "EventPutSuccessCount._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/sum": { + "metric": "EventPutSuccessCount._rate._sum", + "pointInTime": false, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/ChannelSize/rate/avg": { + "metric": "ChannelSize._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/max": { + "metric": "ChannelSize._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/min": { + "metric": "ChannelSize._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/sum": { + "metric": "ChannelSize._rate._sum", + "pointInTime": false, + "temporal": true + } + + } + } + } + ], + "HostComponent": [ + { + "type": "ganglia", + "metrics": { + "default": { + "metrics/flume/flume/CHANNEL/ChannelCapacity": { + "metric": "ChannelCapacity", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeAttemptCount": { + "metric": "EventTakeAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount": { + "metric": "EventTakeSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutAttemptCount": { + "metric": "EventPutAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelFillPercentage": { + "metric": "ChannelFillPercentage", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize": { + "metric": "ChannelSize", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount": { + "metric": "EventPutSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionCreatedCount": { + "metric": "ConnectionCreatedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchCompleteCount": { + "metric": "BatchCompleteCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/EventDrainSuccessCount": { + "metric": "EventDrainSuccessCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/EventDrainAttemptCount": { + "metric": "EventDrainAttemptCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionFailedCount": { + "metric": "ConnectionFailedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchUnderflowCount": { + "metric": "BatchUnderflowCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/ConnectionClosedCount": { + "metric": "ConnectionClosedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SINK/BatchEmptyCount": { + "metric": "BatchEmptyCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendBatchReceivedCount": { + "metric": "AppendBatchReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendAcceptedCount": { + "metric": "AppendAcceptedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/StartTime": { + "metric": "StartTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/OpenConnectionCount": { + "metric": "OpenConnectionCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendBatchAcceptedCount": { + "metric": "AppendBatchAcceptedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/AppendReceivedCount": { + "metric": "AppendReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/EventReceivedCount": { + "metric": "EventReceivedCount", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/StopTime": { + "metric": "StopTime", + "pointInTime": true, + "temporal": true + }, + "metrics/flume/flume/SOURCE/EventAcceptedCount": { + "metric": "EventAcceptedCount", + "pointInTime": true, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/avg": { + "metric": "EventTakeSuccessCount._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/max": { + "metric": "EventTakeSuccessCount._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/min": { + "metric": "EventTakeSuccessCount._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventTakeSuccessCount/rate/sum": { + "metric": "EventTakeSuccessCount._rate._sum", + "pointInTime": false, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/avg": { + "metric": "EventPutSuccessCount._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/max": { + "metric": "EventPutSuccessCount._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/min": { + "metric": "EventPutSuccessCount._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/EventPutSuccessCount/rate/sum": { + "metric": "EventPutSuccessCount._rate._sum", + "pointInTime": false, + "temporal": true + }, + + "metrics/flume/flume/CHANNEL/ChannelSize/rate/avg": { + "metric": "ChannelSize._rate._avg", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/max": { + "metric": "ChannelSize._rate._max", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/min": { + "metric": "ChannelSize._rate._min", + "pointInTime": false, + "temporal": true + }, + "metrics/flume/flume/CHANNEL/ChannelSize/rate/sum": { + "metric": "ChannelSize._rate._sum", + "pointInTime": false, + "temporal": true + } + + } + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py new file mode 100755 index 0000000..df08bb1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/alerts/alert_flume_agent_status.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import socket + +from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names +from resource_management.libraries.functions.flume_agent_helper import get_flume_status + +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +FLUME_CONF_DIR_KEY = '{{flume-env/flume_conf_dir}}' + +FLUME_RUN_DIR_KEY = "run.directory" +FLUME_RUN_DIR_DEFAULT = '/var/run/flume' + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (FLUME_CONF_DIR_KEY,) + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) + + flume_conf_directory = None + if FLUME_CONF_DIR_KEY in configurations: + flume_conf_directory = configurations[FLUME_CONF_DIR_KEY] + + if flume_conf_directory is None: + return (RESULT_CODE_UNKNOWN, ['The Flume configuration directory is a required parameter.']) + + if host_name is None: + host_name = socket.getfqdn() + + # parse script arguments + flume_run_directory = FLUME_RUN_DIR_DEFAULT + if FLUME_RUN_DIR_KEY in parameters: + flume_run_directory = parameters[FLUME_RUN_DIR_KEY] + + processes = get_flume_status(flume_conf_directory, flume_run_directory) + expected_agents = find_expected_agent_names(flume_conf_directory) + + alert_label = '' + alert_state = RESULT_CODE_OK + + if len(processes) == 0 and len(expected_agents) == 0: + alert_label = 'No agents defined on {0}'.format(host_name) + else: + ok = [] + critical = [] + text_arr = [] + + for process in processes: + if not process.has_key('status') or process['status'] == 'NOT_RUNNING': + critical.append(process['name']) + else: + ok.append(process['name']) + + if len(critical) > 0: + text_arr.append("{0} {1} NOT running".format(", ".join(critical), + "is" if len(critical) == 1 else "are")) + + if len(ok) > 0: + text_arr.append("{0} {1} running".format(", ".join(ok), + "is" if len(ok) == 1 else "are")) + + plural = len(critical) > 1 or len(ok) > 1 + alert_label = "Agent{0} {1} {2}".format( + "s" if plural else "", + " and ".join(text_arr), + "on " + host_name) + + alert_state = RESULT_CODE_CRITICAL if len(critical) > 0 else RESULT_CODE_OK + + return (alert_state, [alert_label]) http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py new file mode 100755 index 0000000..8edba7f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume.py @@ -0,0 +1,228 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import glob +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import os +from resource_management import * +from resource_management.libraries.functions.flume_agent_helper import is_flume_process_live +from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names +from resource_management.libraries.functions.flume_agent_helper import await_flume_process_termination +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) +def flume(action = None): + import params + + if action == 'config': + # remove previously defined meta's + for n in find_expected_agent_names(params.flume_conf_dir): + File(os.path.join(params.flume_conf_dir, n, 'ambari-meta.json'), + action = "delete", + ) + + Directory(params.flume_run_dir, + ) + + Directory(params.flume_conf_dir, + create_parents=True, + owner=params.flume_user, + ) + Directory(params.flume_log_dir, owner=params.flume_user) + + flume_agents = {} + if params.flume_conf_content is not None: + flume_agents = build_flume_topology(params.flume_conf_content) + + for agent in flume_agents.keys(): + flume_agent_conf_dir = os.path.join(params.flume_conf_dir, agent) + flume_agent_conf_file = os.path.join(flume_agent_conf_dir, 'flume.conf') + flume_agent_meta_file = os.path.join(flume_agent_conf_dir, 'ambari-meta.json') + flume_agent_log4j_file = os.path.join(flume_agent_conf_dir, 'log4j.properties') + flume_agent_env_file = os.path.join(flume_agent_conf_dir, 'flume-env.sh') + + Directory(flume_agent_conf_dir, + owner=params.flume_user, + ) + + PropertiesFile(flume_agent_conf_file, + properties=flume_agents[agent], + owner=params.flume_user, + mode = 0644) + + File(flume_agent_log4j_file, + content=Template('log4j.properties.j2', agent_name = agent), + owner=params.flume_user, + mode = 0644) + + File(flume_agent_meta_file, + content = json.dumps(ambari_meta(agent, flume_agents[agent])), + owner=params.flume_user, + mode = 0644) + + File(flume_agent_env_file, + owner=params.flume_user, + content=InlineTemplate(params.flume_env_sh_template) + ) + + if params.has_metric_collector: + File(os.path.join(flume_agent_conf_dir, "flume-metrics2.properties"), + owner=params.flume_user, + content=Template("flume-metrics2.properties.j2") + ) + + elif action == 'start': + # desired state for service should be STARTED + if len(params.flume_command_targets) == 0: + _set_desired_state('STARTED') + + # It is important to run this command as a background process. + + flume_base = as_user(format("{flume_bin} agent --name {{0}} --conf {{1}} --conf-file {{2}} {{3}} > {flume_log_dir}/{{4}}.out 2>&1"), params.flume_user, env={'JAVA_HOME': params.java_home}) + " &" + + for agent in cmd_target_names(): + flume_agent_conf_dir = params.flume_conf_dir + os.sep + agent + flume_agent_conf_file = flume_agent_conf_dir + os.sep + "flume.conf" + flume_agent_pid_file = params.flume_run_dir + os.sep + agent + ".pid" + + if not os.path.isfile(flume_agent_conf_file): + continue + + if not is_flume_process_live(flume_agent_pid_file): + # TODO someday make the ganglia ports configurable + extra_args = '' + if params.ganglia_server_host is not None: + extra_args = '-Dflume.monitoring.type=ganglia -Dflume.monitoring.hosts={0}:{1}' + extra_args = extra_args.format(params.ganglia_server_host, '8655') + if params.has_metric_collector: + extra_args = '-Dflume.monitoring.type=org.apache.hadoop.metrics2.sink.flume.FlumeTimelineMetricsSink ' \ + '-Dflume.monitoring.node={0}:{1}' + extra_args = extra_args.format(params.metric_collector_host, params.metric_collector_port) + + flume_cmd = flume_base.format(agent, flume_agent_conf_dir, + flume_agent_conf_file, extra_args, agent) + + Execute(flume_cmd, + wait_for_finish=False, + environment={'JAVA_HOME': params.java_home} + ) + # sometimes startup spawns a couple of threads - so only the first line may count + pid_cmd = as_sudo(('pgrep', '-o', '-u', params.flume_user, '-f', format('^{java_home}.*{agent}.*'))) + \ + " | " + as_sudo(('tee', flume_agent_pid_file)) + " && test ${PIPESTATUS[0]} -eq 0" + Execute(pid_cmd, + logoutput=True, + tries=20, + try_sleep=10) + + pass + elif action == 'stop': + # desired state for service should be INSTALLED + if len(params.flume_command_targets) == 0: + _set_desired_state('INSTALLED') + + pid_files = glob.glob(params.flume_run_dir + os.sep + "*.pid") + + if 0 == len(pid_files): + return + + agent_names = cmd_target_names() + + + for agent in agent_names: + pid_file = format("{flume_run_dir}/{agent}.pid") + + if is_flume_process_live(pid_file): + pid = shell.checked_call(("cat", pid_file), sudo=True)[1].strip() + Execute(("kill", "-15", pid), sudo=True) # kill command has to be a tuple + + if not await_flume_process_termination(pid_file): + raise Fail("Can't stop flume agent: {0}".format(agent)) + + File(pid_file, action = 'delete') + + +def ambari_meta(agent_name, agent_conf): + res = {} + + sources = agent_conf[agent_name + '.sources'].split(' ') + res['sources_count'] = len(sources) + + sinks = agent_conf[agent_name + '.sinks'].split(' ') + res['sinks_count'] = len(sinks) + + channels = agent_conf[agent_name + '.channels'].split(' ') + res['channels_count'] = len(channels) + + return res + + +# define a map of dictionaries, where the key is agent name +# and the dictionary is the name/value pair +def build_flume_topology(content): + + result = {} + agent_names = [] + + for line in content.split('\n'): + rline = line.strip() + if 0 != len(rline) and not rline.startswith('#'): + pair = rline.split('=') + lhs = pair[0].strip() + rhs = pair[1].strip() + + part0 = lhs.split('.')[0] + + if lhs.endswith(".sources"): + agent_names.append(part0) + + if not result.has_key(part0): + result[part0] = {} + + result[part0][lhs] = rhs + + # trim out non-agents + for k in result.keys(): + if not k in agent_names: + del result[k] + + return result + + +def cmd_target_names(): + import params + + if len(params.flume_command_targets) > 0: + return params.flume_command_targets + else: + return find_expected_agent_names(params.flume_conf_dir) + + +def _set_desired_state(state): + import params + File(params.ambari_state_file, + content = state, + ) + +def get_desired_state(): + import params + from resource_management.core import sudo + if os.path.exists(params.ambari_state_file): + return sudo.read_file(params.ambari_state_file) + else: + return 'INSTALLED' http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py new file mode 100755 index 0000000..64cd1c1 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_check.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +class FlumeServiceCheck(Script): + + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) + def service_check(self, env): + import params + env.set_params(params) + if params.security_enabled: + principal_replaced = params.http_principal.replace("_HOST", params.hostname) + Execute(format("{kinit_path_local} -kt {http_keytab} {principal_replaced}"), + user=params.smoke_user) + + Execute(format('env JAVA_HOME={java_home} {flume_bin} version'), + logoutput=True, + tries = 3, + try_sleep = 20) + +if __name__ == "__main__": + FlumeServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py new file mode 100755 index 0000000..d0c6e3b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_handler.py @@ -0,0 +1,145 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import flume_upgrade + +from flume import flume +from flume import get_desired_state + +from resource_management import * +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions.flume_agent_helper import find_expected_agent_names +from resource_management.libraries.functions.flume_agent_helper import get_flume_status +from resource_management.libraries.functions.version import compare_versions, format_stack_version +from resource_management.libraries.functions import Direction + +from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl + +class FlumeHandler(Script): + + def get_component_name(self): + return "flume-server" + + def install(self, env): + import params + self.install_packages(env) + env.set_params(params) + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + flume(action='start') + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + flume(action='stop') + + if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE: + flume_upgrade.post_stop_backup() + #if rolling_restart: + # flume_upgrade.post_stop_backup() + + def configure(self, env): + import params + env.set_params(params) + flume(action='config') + + def status(self, env): + import params + env.set_params(params) + processes = get_flume_status(params.flume_conf_dir, params.flume_run_dir) + expected_agents = find_expected_agent_names(params.flume_conf_dir) + + json = {} + json['processes'] = processes + json['alerts'] = [] + + alert = {} + alert['name'] = 'flume_agent' + alert['label'] = 'Flume Agent process' + + if len(processes) == 0 and len(expected_agents) == 0: + alert['state'] = 'OK' + + if not params.hostname is None: + alert['text'] = 'No agents defined on ' + params.hostname + else: + alert['text'] = 'No agents defined' + + else: + crit = [] + ok = [] + + for proc in processes: + if not proc.has_key('status') or proc['status'] == 'NOT_RUNNING': + crit.append(proc['name']) + else: + ok.append(proc['name']) + + text_arr = [] + + if len(crit) > 0: + text_arr.append("{0} {1} NOT running".format(", ".join(crit), + "is" if len(crit) == 1 else "are")) + + if len(ok) > 0: + text_arr.append("{0} {1} running".format(", ".join(ok), + "is" if len(ok) == 1 else "are")) + + plural = len(crit) > 1 or len(ok) > 1 + alert['text'] = "Agent{0} {1} {2}".format( + "s" if plural else "", + " and ".join(text_arr), + "" if params.hostname is None else "on " + str(params.hostname)) + + alert['state'] = 'CRITICAL' if len(crit) > 0 else 'OK' + + json['alerts'].append(alert) + self.put_structured_out(json) + + # only throw an exception if there are agents defined and there is a + # problem with the processes; if there are no agents defined, then + # the service should report STARTED (green) ONLY if the desired state is started. otherwise, INSTALLED (red) + if len(expected_agents) > 0: + for proc in processes: + if not proc.has_key('status') or proc['status'] == 'NOT_RUNNING': + raise ComponentIsNotRunning() + elif len(expected_agents) == 0 and 'INSTALLED' == get_desired_state(): + raise ComponentIsNotRunning() + + def pre_upgrade_restart(self, env, upgrade_type=None): + import params + env.set_params(params) + + # this function should not execute if the version can't be determined or + # is not at least IOP 4.0.0.0 + if not params.version or compare_versions(format_stack_version(params.version), '4.0.0.0') < 0: + return + + Logger.info("Executing Flume Stack Upgrade pre-restart") + conf_select.select(params.stack_name, "flume", params.version) + stack_select.select("flume-server", params.version) + if params.upgrade_direction == Direction.UPGRADE: + flume_upgrade.pre_start_restore() + +if __name__ == "__main__": + FlumeHandler().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py new file mode 100755 index 0000000..181668b --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/flume_upgrade.py @@ -0,0 +1,94 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +import os +import shutil +import tarfile +import tempfile + +from resource_management.core.logger import Logger +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions import tar_archive + +BACKUP_TEMP_DIR = "flume-upgrade-backup" +BACKUP_CONF_DIR_ARCHIVE = "flume-conf-backup.tar" + +def post_stop_backup(): + """ + Backs up the flume config, config dir, file/spillable channels as part of the + upgrade process. + :return: + """ + Logger.info('Backing up Flume data and configuration before upgrade...') + directoryMappings = _get_directory_mappings() + + absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR) + if not os.path.isdir(absolute_backup_dir): + os.makedirs(absolute_backup_dir) + + for directory in directoryMappings: + if not os.path.isdir(directory): + raise Fail("Unable to backup missing directory {0}".format(directory)) + + archive = os.path.join(absolute_backup_dir, directoryMappings[directory]) + Logger.info('Compressing {0} to {1}'.format(directory, archive)) + + if os.path.exists(archive): + os.remove(archive) + + # backup the directory, following symlinks instead of including them + tar_archive.archive_directory_dereference(archive, directory) + + +def pre_start_restore(): + """ + Restores the flume config, config dir, file/spillable channels to their proper locations + after an upgrade has completed. + :return: + """ + Logger.info('Restoring Flume data and configuration after upgrade...') + directoryMappings = _get_directory_mappings() + + for directory in directoryMappings: + archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR, + directoryMappings[directory]) + + if os.path.isfile(archive): + Logger.info('Extracting {0} to {1}'.format(archive, directory)) + tarball = None + try: + tarball = tarfile.open(archive, "r") + tarball.extractall(directory) + finally: + if tarball: + tarball.close() + + # cleanup + if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)): + shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)) + +def _get_directory_mappings(): + """ + Gets a dictionary of directory to archive name that represents the + directories that need to be backed up and their output tarball archive targets + :return: the dictionary of directory to tarball mappings + """ + import params + + return { params.flume_conf_dir : BACKUP_CONF_DIR_ARCHIVE} http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py new file mode 100755 index 0000000..f9addb9 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py @@ -0,0 +1,101 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.functions.constants import Direction +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions import format +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions.default import default +from resource_management.libraries.script.script import Script + +from params_linux import * + +config = Script.get_config() + +stack_name = default("/hostLevelParams/stack_name", None) +upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) +host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) + +# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade +version = default("/commandParams/version", None) + +user_group = config['configurations']['cluster-env']['user_group'] +proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] + +security_enabled = False + +stack_version_unformatted = str(config['hostLevelParams']['stack_version']) +iop_stack_version = format_stack_version(stack_version_unformatted) + +# hadoop default parameters +flume_bin = '/usr/iop/current/flume-server/bin/flume-ng' + +# hadoop parameters for 2.2+ +flume_hive_home = '/usr/iop/current/hive-metastore' +flume_hcat_home = '/usr/iop/current/hive-webhcat' + +# flume agent runtime for management +flume_log_dir = config['configurations']['flume-env']['flume_log_dir'] + +# flume_run_dir is a new parameter added since Ambari 2.1 in IOP 4.1 +# The default value must be the same as the value of "flume_run_dir" in params.py of Ambari 1.7 in IOP 4.0 +# See ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params.py +FLUME_RUN_DIR_DEFAULT = '/var/run/flume' +if (('flume-env' in config['configurations']) and ('flume_run_dir' in config['configurations']['flume-env'])): + flume_run_dir = config['configurations']['flume-env']['flume_run_dir'] +else: + flume_run_dir = FLUME_RUN_DIR_DEFAULT + +java_home = config['hostLevelParams']['java_home'] +ambari_state_file = format("{flume_run_dir}/ambari-state.txt") + +if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']: + flume_user = config['configurations']['flume-env']['flume_user'] + +if (('flume-conf' in config['configurations']) and('content' in config['configurations']['flume-conf'])): + flume_conf_content = config['configurations']['flume-conf']['content'] +else: + flume_conf_content = None + +if (('flume-log4j' in config['configurations']) and ('content' in config['configurations']['flume-log4j'])): + flume_log4j_content = config['configurations']['flume-log4j']['content'] +else: + flume_log4j_content = None + +targets = default('/commandParams/flume_handler', None) +flume_command_targets = [] if targets is None else targets.split(',') + +flume_env_sh_template = config['configurations']['flume-env']['content'] + +ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) +ganglia_server_host = None +if 0 != len(ganglia_server_hosts): + ganglia_server_host = ganglia_server_hosts[0] + +hostname = None +if config.has_key('hostname'): + hostname = config['hostname'] + +ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", []) +has_metric_collector = not len(ams_collector_hosts) == 0 +if has_metric_collector: + metric_collector_host = ams_collector_hosts[0] + metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188") + if metric_collector_port and metric_collector_port.find(':') != -1: + metric_collector_port = metric_collector_port.split(':')[1] + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py new file mode 100755 index 0000000..7ecbe27 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/scripts/params_linux.py @@ -0,0 +1,30 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.libraries.functions import format +from resource_management.libraries.script.script import Script + +# server configurations +config = Script.get_config() + +flume_conf_dir = '/usr/iop/current/flume-server/conf' + +flume_user = 'flume' +flume_group = 'flume' +if 'flume-env' in config['configurations'] and 'flume_user' in config['configurations']['flume-env']: + flume_user = config['configurations']['flume-env']['flume_user'] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2 new file mode 100755 index 0000000..bc18043 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume-metrics2.properties.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +collector={{metric_collector_host}} +port={{metric_collector_port}} +collectionFrequency=60000 +maxRowCacheSize=10000 +sendInterval=59000 + +# Metric names having type COUNTER +counters=EventTakeSuccessCount,EventPutSuccessCount,EventTakeAttemptCount,EventPutAttemptCount http://git-wip-us.apache.org/repos/asf/ambari/blob/1863c3b9/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2 b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2 new file mode 100755 index 0000000..4dee67f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/FLUME/package/templates/flume.conf.j2 @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# flume.conf: Add your flume configuration here and start flume +# Note if you are using the Windows srvice or Unix service +# provided by the HDP distribution, they will assume the +# agent's name in this file to be 'a1' +# +{{flume_agent_conf_content}}
