AMBARI-8779: Common Services: Refactor HDP-2.1 FALCON, STORM, TEZ services (Jayush Luniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/dc534389 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/dc534389 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/dc534389 Branch: refs/heads/trunk Commit: dc534389a7fdf54cec2be3ad939ff59c1e33d636 Parents: ef7fe6b Author: Jayush Luniya <[email protected]> Authored: Thu Dec 18 21:31:43 2014 -0800 Committer: Jayush Luniya <[email protected]> Committed: Thu Dec 18 21:31:43 2014 -0800 ---------------------------------------------------------------------- .../FALCON/0.5.0.2.1/alerts.json | 58 + .../0.5.0.2.1/configuration/falcon-env.xml | 110 ++ .../configuration/falcon-runtime.properties.xml | 47 + .../configuration/falcon-startup.properties.xml | 207 ++++ .../0.5.0.2.1/configuration/oozie-site.xml | 176 +++ .../FALCON/0.5.0.2.1/metainfo.xml | 117 ++ .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 124 ++ .../0.5.0.2.1/package/scripts/falcon_client.py | 38 + .../0.5.0.2.1/package/scripts/falcon_server.py | 61 + .../FALCON/0.5.0.2.1/package/scripts/params.py | 90 ++ .../0.5.0.2.1/package/scripts/service_check.py | 40 + .../0.5.0.2.1/package/scripts/status_params.py | 24 + .../package/templates/client.properties.j2 | 42 + .../package/templates/runtime.properties.j2 | 50 + .../common-services/STORM/0.9.1.2.1/alerts.json | 189 +++ .../STORM/0.9.1.2.1/configuration/storm-env.xml | 56 + .../0.9.1.2.1/configuration/storm-site.xml | 580 ++++++++++ .../STORM/0.9.1.2.1/metainfo.xml | 126 ++ .../STORM/0.9.1.2.1/metrics.json | 1077 ++++++++++++++++++ .../STORM/0.9.1.2.1/package/files/wordCount.jar | Bin 0 -> 690588 bytes .../0.9.1.2.1/package/scripts/drpc_server.py | 58 + .../STORM/0.9.1.2.1/package/scripts/nimbus.py | 57 + .../0.9.1.2.1/package/scripts/nimbus_prod.py | 55 + .../STORM/0.9.1.2.1/package/scripts/params.py | 90 ++ .../STORM/0.9.1.2.1/package/scripts/rest_api.py | 58 + .../STORM/0.9.1.2.1/package/scripts/service.py | 84 ++ .../0.9.1.2.1/package/scripts/service_check.py | 49 + .../0.9.1.2.1/package/scripts/status_params.py | 36 + .../STORM/0.9.1.2.1/package/scripts/storm.py | 98 ++ .../0.9.1.2.1/package/scripts/supervisor.py | 61 + .../package/scripts/supervisor_prod.py | 57 + .../package/scripts/supervisord_service.py | 32 + .../0.9.1.2.1/package/scripts/ui_server.py | 58 + .../0.9.1.2.1/package/scripts/yaml_utils.py | 49 + .../package/templates/client_jaas.conf.j2 | 24 + .../0.9.1.2.1/package/templates/config.yaml.j2 | 65 ++ .../templates/storm-metrics2.properties.j2 | 21 + .../0.9.1.2.1/package/templates/storm.yaml.j2 | 66 ++ .../package/templates/storm_jaas.conf.j2 | 45 + .../package/templates/worker-launcher.cfg.j2 | 19 + .../TEZ/0.4.0.2.1/configuration/tez-env.xml | 46 + .../TEZ/0.4.0.2.1/configuration/tez-site.xml | 215 ++++ .../common-services/TEZ/0.4.0.2.1/metainfo.xml | 73 ++ .../TEZ/0.4.0.2.1/package/scripts/params.py | 67 ++ .../TEZ/0.4.0.2.1/package/scripts/tez.py | 61 + .../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 49 + .../stacks/HDP/2.1/services/FALCON/alerts.json | 58 - .../FALCON/configuration/falcon-env.xml | 110 -- .../configuration/falcon-runtime.properties.xml | 47 - .../configuration/falcon-startup.properties.xml | 207 ---- .../FALCON/configuration/oozie-site.xml | 176 --- .../stacks/HDP/2.1/services/FALCON/metainfo.xml | 93 +- .../services/FALCON/package/scripts/falcon.py | 124 -- .../FALCON/package/scripts/falcon_client.py | 38 - .../FALCON/package/scripts/falcon_server.py | 61 - .../services/FALCON/package/scripts/params.py | 90 -- .../FALCON/package/scripts/service_check.py | 40 - .../FALCON/package/scripts/status_params.py | 24 - .../package/templates/client.properties.j2 | 42 - .../package/templates/runtime.properties.j2 | 50 - .../stacks/HDP/2.1/services/STORM/alerts.json | 189 --- .../services/STORM/configuration/storm-env.xml | 56 - .../services/STORM/configuration/storm-site.xml | 580 ---------- .../stacks/HDP/2.1/services/STORM/metainfo.xml | 101 +- .../stacks/HDP/2.1/services/STORM/metrics.json | 1077 ------------------ .../services/STORM/package/files/wordCount.jar | Bin 690588 -> 0 bytes .../STORM/package/scripts/drpc_server.py | 58 - .../services/STORM/package/scripts/nimbus.py | 57 - .../STORM/package/scripts/nimbus_prod.py | 55 - .../services/STORM/package/scripts/params.py | 90 -- .../services/STORM/package/scripts/rest_api.py | 58 - .../services/STORM/package/scripts/service.py | 84 -- .../STORM/package/scripts/service_check.py | 49 - .../STORM/package/scripts/status_params.py | 36 - .../2.1/services/STORM/package/scripts/storm.py | 98 -- .../STORM/package/scripts/supervisor.py | 61 - .../STORM/package/scripts/supervisor_prod.py | 57 - .../package/scripts/supervisord_service.py | 32 - .../services/STORM/package/scripts/ui_server.py | 58 - .../STORM/package/scripts/yaml_utils.py | 49 - .../STORM/package/templates/client_jaas.conf.j2 | 24 - .../STORM/package/templates/config.yaml.j2 | 65 -- .../templates/storm-metrics2.properties.j2 | 21 - .../STORM/package/templates/storm.yaml.j2 | 66 -- .../STORM/package/templates/storm_jaas.conf.j2 | 45 - .../package/templates/worker-launcher.cfg.j2 | 19 - .../2.1/services/TEZ/configuration/tez-env.xml | 46 - .../2.1/services/TEZ/configuration/tez-site.xml | 215 ---- .../stacks/HDP/2.1/services/TEZ/metainfo.xml | 49 +- .../2.1/services/TEZ/package/scripts/params.py | 67 -- .../HDP/2.1/services/TEZ/package/scripts/tez.py | 61 - .../services/TEZ/package/scripts/tez_client.py | 49 - .../stacks/2.1/FALCON/test_falcon_client.py | 9 +- .../stacks/2.1/FALCON/test_falcon_server.py | 20 +- .../stacks/2.1/FALCON/test_service_check.py | 14 +- .../stacks/2.1/STORM/test_service_check.py | 10 +- .../python/stacks/2.1/STORM/test_storm_base.py | 3 + .../stacks/2.1/STORM/test_storm_drpc_server.py | 36 +- .../2.1/STORM/test_storm_jaas_configuration.py | 24 +- .../stacks/2.1/STORM/test_storm_nimbus.py | 36 +- .../stacks/2.1/STORM/test_storm_nimbus_prod.py | 36 +- .../2.1/STORM/test_storm_rest_api_service.py | 36 +- .../stacks/2.1/STORM/test_storm_supervisor.py | 36 +- .../2.1/STORM/test_storm_supervisor_prod.py | 36 +- .../stacks/2.1/STORM/test_storm_ui_server.py | 36 +- .../python/stacks/2.1/TEZ/test_tez_client.py | 9 +- 106 files changed, 5040 insertions(+), 4838 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json new file mode 100644 index 0000000..c34093f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json @@ -0,0 +1,58 @@ +{ + "FALCON": { + "service": [], + "FALCON_SERVER": [ + { + "name": "falcon_server_process", + "label": "Falcon Server Process", + "description": "This host-level alert is triggered if the individual Falcon server process cannot be established to be up and listening on the network.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "PORT", + "uri": "{{falcon-env/falcon_port}}", + "default_port": 15000, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + }, + { + "name": "falcon_server_webui", + "label": "Falcon Server Web UI", + "description": "This host-level alert is triggered if the Falcon Server Web UI is unreachable.", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "WEB", + "uri": { + "http": "{{falcon-env/falcon_port}}", + "default_port": 15000 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.4f} seconds" + }, + "warning":{ + "text": "HTTP {0} response in {2:.4f} seconds" + }, + "critical": { + "text": "Connection failed to {1}" + } + } + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml new file mode 100644 index 0000000..0eaad86 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml @@ -0,0 +1,110 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>falcon_user</name> + <value>falcon</value> + <property-type>USER</property-type> + <description>Falcon user.</description> + </property> + <property> + <name>falcon_port</name> + <value>15000</value> + <description>Port the Falcon Server listens on.</description> + </property> + <property> + <name>falcon_log_dir</name> + <value>/var/log/falcon</value> + <description>Falcon log directory.</description> + </property> + <property> + <name>falcon_pid_dir</name> + <value>/var/run/falcon</value> + <description>Falcon pid-file directory.</description> + </property> + <property> + <name>falcon_local_dir</name> + <value>/hadoop/falcon</value> + <description>Directory where Falcon data, such as activemq data, is stored.</description> + </property> + <!--embeddedmq properties--> + <property> + <name>falcon.embeddedmq.data</name> + <value>/hadoop/falcon/embeddedmq/data</value> + <description>Directory in which embeddedmq data is stored.</description> + </property> + <property> + <name>falcon.embeddedmq</name> + <value>true</value> + <description>Whether embeddedmq is enabled or not.</description> + </property> + <property> + <name>falcon.emeddedmq.port</name> + <value>61616</value> + <description>Port that embeddedmq will listen on.</description> + </property> + + <!-- falcon-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for falcon-env.sh file</description> + <value> +# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path +export JAVA_HOME={{java_home}} + +# any additional java opts you want to set. This will apply to both client and server operations +#export FALCON_OPTS= + +# any additional java opts that you want to set for client only +#export FALCON_CLIENT_OPTS= + +# java heap size we want to set for the client. Default is 1024MB +#export FALCON_CLIENT_HEAP= + +# any additional opts you want to set for prisim service. +#export FALCON_PRISM_OPTS= + +# java heap size we want to set for the prisim service. Default is 1024MB +#export FALCON_PRISM_HEAP= + +# any additional opts you want to set for falcon service. +export FALCON_SERVER_OPTS="-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}" + +# java heap size we want to set for the falcon server. Default is 1024MB +#export FALCON_SERVER_HEAP= + +# What is is considered as falcon home dir. Default is the base location of the installed software +#export FALCON_HOME_DIR= + +# Where log files are stored. Defatult is logs directory under the base install location +export FALCON_LOG_DIR={{falcon_log_dir}} + +# Where pid files are stored. Defatult is logs directory under the base install location +export FALCON_PID_DIR={{falcon_pid_dir}} + +# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location +export FALCON_DATA_DIR={{falcon_embeddedmq_data}} + +# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir. +#export FALCON_EXPANDED_WEBAPP_DIR= + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-runtime.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-runtime.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-runtime.properties.xml new file mode 100644 index 0000000..94c8755 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-runtime.properties.xml @@ -0,0 +1,47 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <property> + <name>*.domain</name> + <value>${falcon.app.type}</value> + <description></description> + </property> + <property> + <name>*.log.cleanup.frequency.minutes.retention</name> + <value>hours(6)</value> + <description></description> + </property> + <property> + <name>*.log.cleanup.frequency.hours.retention</name> + <value>minutes(1)</value> + <description></description> + </property> + <property> + <name>*.log.cleanup.frequency.days.retention</name> + <value>days(7)</value> + <description></description> + </property> + <property> + <name>*.log.cleanup.frequency.months.retention</name> + <value>months(3)</value> + <description></description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml new file mode 100644 index 0000000..7459429 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-startup.properties.xml @@ -0,0 +1,207 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <!--advanced properties--> + <property> + <name>*.workflow.engine.impl</name> + <value>org.apache.falcon.workflow.engine.OozieWorkflowEngine</value> + <description></description> + </property> + <property> + <name>*.oozie.process.workflow.builder</name> + <value>org.apache.falcon.workflow.OozieProcessWorkflowBuilder</value> + <description></description> + </property> + <property> + <name>*.oozie.feed.workflow.builder</name> + <value>org.apache.falcon.workflow.OozieFeedWorkflowBuilder</value> + <description></description> + </property> + <property> + <name>*.SchedulableEntityManager.impl</name> + <value>org.apache.falcon.resource.SchedulableEntityManager</value> + <description></description> + </property> + <property> + <name>*.ConfigSyncService.impl</name> + <value>org.apache.falcon.resource.ConfigSyncService</value> + <description></description> + </property> + <property> + <name>*.ProcessInstanceManager.impl</name> + <value>org.apache.falcon.resource.InstanceManager</value> + <description></description> + </property> + <property> + <name>*.catalog.service.impl</name> + <value>org.apache.falcon.catalog.HiveCatalogService</value> + <description></description> + </property> + <property> + <name>*.application.services</name> + <value>org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.service.LogCleanupService + </value> + <description></description> + </property> + <property> + <name>*.configstore.listeners</name> + <value>org.apache.falcon.entity.v0.EntityGraph,\ + org.apache.falcon.entity.ColoClusterRelation,\ + org.apache.falcon.group.FeedGroupMap,\ + org.apache.falcon.service.SharedLibraryHostingService + </value> + <description></description> + </property> + <property> + <name>*.broker.impl.class</name> + <value>org.apache.activemq.ActiveMQConnectionFactory</value> + <description></description> + </property> + <property> + <name>*.shared.libs</name> + <value>activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms</value> + <description></description> + </property> + <!--common properties--> + <property> + <name>*.domain</name> + <value>${falcon.app.type}</value> + <description></description> + </property> + <property> + <name>*.config.store.uri</name> + <value>file:///hadoop/falcon/store</value> + <description>Location to store user entity configurations</description> + </property> + <property> + <name>*.system.lib.location</name> + <value>${falcon.home}/server/webapp/${falcon.app.type}/WEB-INF/lib</value> + <description>Location of libraries that is shipped to Hadoop</description> + </property> + <property> + <name>*.retry.recorder.path</name> + <value>${falcon.log.dir}/retry</value> + <description></description> + </property> + <property> + <name>*.falcon.cleanup.service.frequency</name> + <value>days(1)</value> + <description></description> + </property> + <property> + <name>*.broker.url</name> + <value>tcp://localhost:61616</value> + <description>Default Active MQ url</description> + </property> + <property> + <name>*.broker.ttlInMins</name> + <value>4320</value> + <description></description> + </property> + <property> + <name>*.entity.topic</name> + <value>FALCON.ENTITY.TOPIC</value> + <description></description> + </property> + <property> + <name>*.max.retry.failure.count</name> + <value>1</value> + <description></description> + </property> + <property> + <name>*.internal.queue.size</name> + <value>1000</value> + <description></description> + </property> + <!--properties without default values--> + <property> + <name>*.falcon.http.authentication.cookie.domain</name> + <value>EXAMPLE.COM</value> + <description></description> + </property> + <property> + <name>*.falcon.http.authentication.blacklisted.users</name> + <value></value> + <description>Comma separated list of black listed users</description> + </property> + <!--authentication properties--> + <property> + <name>*.falcon.authentication.type</name> + <value>simple</value> + <description></description> + </property> + <property> + <name>*.falcon.http.authentication.type</name> + <value>simple</value> + <description></description> + </property> + <property> + <name>*.falcon.http.authentication.token.validity</name> + <value>36000</value> + <description></description> + </property> + <property> + <name>*.falcon.http.authentication.signature.secret</name> + <value>falcon</value> + <description></description> + </property> + <property> + <name>*.falcon.http.authentication.simple.anonymous.allowed</name> + <value>true</value> + <description>Indicates if anonymous requests are allowed when using 'simple' authentication</description> + </property> + <property> + <name>*.falcon.http.authentication.kerberos.name.rules</name> + <value>DEFAULT</value> + <description>The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's KerberosName for more details.</description> + </property> + <!--kerberos params, must be set during security enabling--> + <property> + <name>*.falcon.service.authentication.kerberos.principal</name> + <value>falcon/[email protected]</value> + <description></description> + </property> + <property> + <name>*.falcon.service.authentication.kerberos.keytab</name> + <value>/etc/security/keytabs/falcon.service.keytab</value> + <description></description> + </property> + <property> + <name>*.dfs.namenode.kerberos.principal</name> + <value>nn/[email protected]</value> + <description>name node principal to talk to config store</description> + </property> + <property> + <name>*.falcon.http.authentication.kerberos.principal</name> + <value>HTTP/[email protected]</value> + <description>Indicates the Kerberos principal to be used for HTTP endpoint</description> + </property> + <property> + <name>*.falcon.http.authentication.kerberos.keytab</name> + <value>/etc/security/keytabs/spnego.service.keytab</value> + <description>Location of the keytab file with the credentials for the HTTP principal</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml new file mode 100644 index 0000000..c6962f3 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml @@ -0,0 +1,176 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration supports_final="true"> + <property> + <name>oozie.service.ELService.ext.functions.coord-job-submit-instances</name> + <value> + now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo, + today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-action-create-inst</name> + <value> + now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst, + today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst, + latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-action-create</name> + <value> + now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, + today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear, + latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-job-submit-data</name> + <value> + now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo, + today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo, + dataIn=org.apache.oozie.extensions.OozieELExtensions#ph1_dataIn_echo, + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-action-start</name> + <value> + now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, + today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, + yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, + currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth, + lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth, + currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear, + lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear, + latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest, + future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future, + dataIn=org.apache.oozie.extensions.OozieELExtensions#ph3_dataIn, + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime, + dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset, + formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-sla-submit</name> + <value> + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + </description> + </property> + + <property> + <name>oozie.service.ELService.ext.functions.coord-sla-create</name> + <value> + instanceTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime, + user=org.apache.oozie.coord.CoordELFunctions#coord_user + </value> + <description> + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + </description> + </property> + + <property> + <name>oozie.service.HadoopAccessorService.supported.filesystems</name> + <value>*</value> + <description> + Enlist the different filesystems supported for federation. If wildcard "*" is specified, then ALL file schemes will be allowed. + </description> + </property> + + <!--web ui should add following properties to oozie site accordingly to FALCON_USER--> + <!--<property>--> + <!--<name>oozie.service.ProxyUserService.proxyuser.#FALCON_USER#.hosts</name>--> + <!--<value>*</value>--> + <!--<description>Falcon proxyuser hosts</description>--> + <!--</property>--> + + <!--<property>--> + <!--<name>oozie.service.ProxyUserService.proxyuser.#FALCON_USER#.groups</name>--> + <!--<value>*</value>--> + <!--<description>Falcon proxyuser groups</description>--> + <!--</property>--> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml new file mode 100644 index 0000000..78336e6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/metainfo.xml @@ -0,0 +1,117 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>FALCON</name> + <displayName>Falcon</displayName> + <comment>Data management and processing platform</comment> + <version>0.5.0.2.1</version> + <components> + <component> + <name>FALCON_CLIENT</name> + <displayName>Falcon Client</displayName> + <category>CLIENT</category> + <cardinality>1+</cardinality> + <commandScript> + <script>scripts/falcon_client.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + <configFiles> + <configFile> + <type>env</type> + <fileName>falcon-env.sh</fileName> + <dictionaryName>falcon-env</dictionaryName> + </configFile> + <configFile> + <type>properties</type> + <fileName>runtime.properties</fileName> + <dictionaryName>falcon-runtime.properties</dictionaryName> + </configFile> + <configFile> + <type>properties</type> + <fileName>startup.properties</fileName> + <dictionaryName>falcon-startup.properties</dictionaryName> + </configFile> + </configFiles> + </component> + <component> + <name>FALCON_SERVER</name> + <displayName>Falcon Server</displayName> + <category>MASTER</category> + <cardinality>1</cardinality> + <dependencies> + <dependency> + <name>OOZIE/OOZIE_SERVER</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>OOZIE/OOZIE_CLIENT</name> + <scope>cluster</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/falcon_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>falcon</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <requiredServices> + <service>OOZIE</service> + </requiredServices> + + <configuration-dependencies> + <config-type>falcon-env</config-type> + <config-type>falcon-startup.properties</config-type> + <config-type>falcon-runtime.properties</config-type> + </configuration-dependencies> + + <excluded-config-types> + <config-type>oozie-site</config-type> + </excluded-config-types> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py new file mode 100644 index 0000000..a903e5c --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -0,0 +1,124 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import os.path + +def falcon(type, action = None): + import params + if action == 'config': + Directory(params.falcon_pid_dir, + owner=params.falcon_user + ) + Directory(params.falcon_log_dir, + owner=params.falcon_user, + recursive=True + ) + Directory(params.falcon_webapp_dir, + owner=params.falcon_user + ) + Directory(params.falcon_home, + owner=params.falcon_user + ) + Directory(params.falcon_conf_dir_prefix, + mode=0755 + ) + Directory(params.falcon_conf_dir, + owner=params.falcon_user, + recursive=True + ) + File(params.falcon_conf_dir + '/falcon-env.sh', + content=InlineTemplate(params.falcon_env_sh_template), + owner=params.falcon_user + ) + File(params.falcon_conf_dir + '/client.properties', + content=Template('client.properties.j2'), + mode=0644, + owner=params.falcon_user + ) + PropertiesFile(params.falcon_conf_dir + '/runtime.properties', + properties=params.falcon_runtime_properties, + mode=0644, + owner=params.falcon_user + ) + PropertiesFile(params.falcon_conf_dir + '/startup.properties', + properties=params.falcon_startup_properties, + mode=0644, + owner=params.falcon_user + ) + + if params.falcon_graph_storage_directory: + Directory(params.falcon_graph_storage_directory, + owner=params.falcon_user, + group=params.user_group, + mode=0775, + recursive=True, + recursive_permission=True + ) + + if params.falcon_graph_serialize_path: + Directory(params.falcon_graph_serialize_path, + owner=params.falcon_user, + group=params.user_group, + mode=0775, + recursive=True, + recursive_permission=True + ) + + if type == 'server': + if action == 'config': + if params.store_uri[0:4] == "hdfs": + params.HdfsDirectory(params.store_uri, + action="create_delayed", + owner=params.falcon_user, + mode=0755 + ) + params.HdfsDirectory(params.flacon_apps_dir, + action="create_delayed", + owner=params.falcon_user, + mode=0777#TODO change to proper mode + ) + params.HdfsDirectory(None, action="create") + Directory(params.falcon_local_dir, + owner=params.falcon_user, + recursive=True, + recursive_permission=True + ) + if params.falcon_embeddedmq_enabled == True: + Directory(os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")), + owner=params.falcon_user + ) + Directory(params.falcon_embeddedmq_data, + owner=params.falcon_user, + recursive=True + ) + + if action == 'start': + Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'), + user=params.falcon_user, + path=params.hadoop_bin_dir + ) + if action == 'stop': + Execute(format('{falcon_home}/bin/falcon-stop'), + user=params.falcon_user, + path=params.hadoop_bin_dir + ) + File(params.server_pid_file, + action='delete' + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py new file mode 100644 index 0000000..fd8c005 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py @@ -0,0 +1,38 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from falcon import falcon + +class FalconClient(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + + env.set_params(params) + falcon('client', action='config') + + def status(self, env): + raise ClientComponentHasNoStatus() + +if __name__ == "__main__": + FalconClient().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py new file mode 100644 index 0000000..0460460 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py @@ -0,0 +1,61 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from falcon import falcon + +class FalconServer(Script): + def install(self, env): + import params + + self.install_packages(env) + env.set_params(params) + + def start(self, env): + import params + + env.set_params(params) + self.configure(env) + + falcon('server', action='start') + + def stop(self, env): + import params + + env.set_params(params) + + falcon('server', action='stop') + + + def configure(self, env): + import params + + env.set_params(params) + + falcon('server', action='config') + + def status(self, env): + import status_params + + env.set_params(status_params) + check_process_status(status_params.server_pid_file) + + +if __name__ == "__main__": + FalconServer().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py new file mode 100644 index 0000000..19b647e --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py @@ -0,0 +1,90 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions +from resource_management import * + +from status_params import * + +config = Script.get_config() + +hdp_stack_version = str(config['hostLevelParams']['stack_version']) +hdp_stack_version = format_hdp_stack_version(hdp_stack_version) + +#hadoop params +if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: + hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin" + falcon_webapp_dir = "/usr/hdp/current/falcon-client/webapp" + falcon_home = "/usr/hdp/current/falcon-client" +else: + hadoop_bin_dir = "/usr/bin" + falcon_webapp_dir = '/var/lib/falcon/webapp' + falcon_home = '/usr/lib/falcon' + +hadoop_conf_dir = "/etc/hadoop/conf" +falcon_conf_dir_prefix = "/etc/falcon" +falcon_conf_dir = format("{falcon_conf_dir_prefix}/conf") +oozie_user = config['configurations']['oozie-env']['oozie_user'] +falcon_user = config['configurations']['falcon-env']['falcon_user'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] + +user_group = config['configurations']['cluster-env']['user_group'] +proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] + +java_home = config['hostLevelParams']['java_home'] +falcon_local_dir = config['configurations']['falcon-env']['falcon_local_dir'] +falcon_log_dir = config['configurations']['falcon-env']['falcon_log_dir'] + +# falcon-startup.properties +store_uri = config['configurations']['falcon-startup.properties']['*.config.store.uri'] +# If these properties are present, the directories need to be created. +falcon_graph_storage_directory = default("/configurations/falcon-startup.properties/*.falcon.graph.storage.directory", None) # explicitly set in HDP 2.2 and higher +falcon_graph_serialize_path = default("/configurations/falcon-startup.properties/*.falcon.graph.serialize.path", None) # explicitly set in HDP 2.2 and higher + +falcon_embeddedmq_data = config['configurations']['falcon-env']['falcon.embeddedmq.data'] +falcon_embeddedmq_enabled = config['configurations']['falcon-env']['falcon.embeddedmq'] +falcon_emeddedmq_port = config['configurations']['falcon-env']['falcon.emeddedmq.port'] + +falcon_host = config['clusterHostInfo']['falcon_server_hosts'][0] +falcon_port = config['configurations']['falcon-env']['falcon_port'] +falcon_runtime_properties = config['configurations']['falcon-runtime.properties'] +falcon_startup_properties = config['configurations']['falcon-startup.properties'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +falcon_env_sh_template = config['configurations']['falcon-env']['content'] + +flacon_apps_dir = '/apps/falcon' +#for create_hdfs_directory +security_enabled = config['configurations']['cluster-env']['security_enabled'] +hostname = config["hostname"] +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] +kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) +import functools +#create partial functions with common arguments for every HdfsDirectory call +#to create hdfs directory we need to call params.HdfsDirectory in code +HdfsDirectory = functools.partial( + HdfsDirectory, + conf_dir=hadoop_conf_dir, + hdfs_user=hdfs_user, + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + bin_dir = hadoop_bin_dir +) http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/service_check.py new file mode 100644 index 0000000..bf3b1dd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/service_check.py @@ -0,0 +1,40 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + + +class FalconServiceCheck(Script): + + def service_check(self, env): + import params + + env.set_params(params) + if params.security_enabled: + Execute(format("{kinit_path_local} -kt {smokeuser_keytab} {smoke_user}"), + user=params.smoke_user) + Execute(format("{falcon_home}/bin/falcon admin -version"), + user=params.smoke_user, + logoutput=True, + tries = 3, + try_sleep = 20 + ) + +if __name__ == "__main__": + FalconServiceCheck().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py new file mode 100644 index 0000000..6ebb35f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/status_params.py @@ -0,0 +1,24 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +config = Script.get_config() +falcon_pid_dir = config['configurations']['falcon-env']['falcon_pid_dir'] +server_pid_file = format('{falcon_pid_dir}/falcon.pid') http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/client.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/client.properties.j2 b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/client.properties.j2 new file mode 100644 index 0000000..63749db --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/client.properties.j2 @@ -0,0 +1,42 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +######################################################################### +########## This is used for falcon packaging only. ################### +## Uses default port. Please change if configured for non-default port ## +######################################################################### + +falcon.url=http://{{falcon_host}}:{{falcon_port}}/ http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/runtime.properties.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/runtime.properties.j2 b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/runtime.properties.j2 new file mode 100644 index 0000000..06f17de --- /dev/null +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/templates/runtime.properties.j2 @@ -0,0 +1,50 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#################################################### +#### This is used for falcon packaging only. #### +#################################################### + +*.domain=${falcon.app.type} + +*.log.cleanup.frequency.minutes.retention=hours(6) +*.log.cleanup.frequency.hours.retention=minutes(1) +*.log.cleanup.frequency.days.retention=days(7) +*.log.cleanup.frequency.months.retention=months(3) +#### To configure falcon servers with prism #### +#*.all.colos=<comma separated list of colos where falcon servers are installed> +#*.falcon.<colo>.endpoint=<falcon server endpoint> + http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json new file mode 100644 index 0000000..4800fcb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json @@ -0,0 +1,189 @@ +{ + "STORM": { + "service": [ + { + "name": "storm_supervisor_process_percent", + "label": "Percent Supervisors Available", + "interval": 1, + "scope": "SERVICE", + "enabled": true, + "source": { + "type": "AGGREGATE", + "alert_name": "storm_supervisor_process", + "reporting": { + "ok": { + "text": "affected: [{1}], total: [{0}]" + }, + "warning": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.1 + }, + "critical": { + "text": "affected: [{1}], total: [{0}]", + "value": 0.3 + } + } + } + } + ], + "STORM_UI_SERVER": [ + { + "name": "storm_server_process", + "label": "Storm Server Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/ui.port}}", + "default_port": 8744, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + }, + { + "name": "storm_webui", + "label": "Storm Web UI", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "WEB", + "uri": { + "http": "{{storm-site/ui.port}}" + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.4f} seconds" + }, + "warning":{ + "text": "HTTP {0} response in {2:.4f} seconds" + }, + "critical": { + "text": "Connection failed to {1}" + } + } + } + } + ], + "NIMBUS": [ + { + "name": "storm_nimbus_process", + "label": "Nimbus Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/nimbus.thrift.port}}", + "default_port": 6627, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "DRPC_SERVER": [ + { + "name": "storm_drpc_server", + "label": "DRPC Server Process", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "{{storm-site/drpc.port}}", + "default_port": 3772, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "STORM_REST_API": [ + { + "name": "storm_rest_api", + "label": "Storm REST API", + "interval": 1, + "scope": "ANY", + "enabled": true, + "source": { + "type": "PORT", + "uri": "8745", + "default_port": 8745, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ], + "SUPERVISOR": [ + { + "name": "storm_supervisor_process", + "label": "Supervisor Process", + "interval": 1, + "scope": "HOST", + "enabled": true, + "source": { + "type": "PORT", + "uri": "56431", + "default_port": 56431, + "reporting": { + "ok": { + "text": "TCP OK - {0:.3f}s response on port {1}" + }, + "warning": { + "text": "TCP OK - {0:.3f}s response on port {1}", + "value": 1.5 + }, + "critical": { + "text": "Connection failed: {0} to {1}:{2}", + "value": 5.0 + } + } + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/dc534389/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-env.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-env.xml new file mode 100644 index 0000000..c663116 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-env.xml @@ -0,0 +1,56 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>storm_user</name> + <value>storm</value> + <property-type>USER</property-type> + <description></description> + </property> + <property> + <name>storm_log_dir</name> + <value>/var/log/storm</value> + <description></description> + </property> + <property> + <name>storm_pid_dir</name> + <value>/var/run/storm</value> + <description></description> + </property> + + <!-- storm-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for storm-env.sh file</description> + <value> +#!/bin/bash + +# Set Storm specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java64_home}} + +# export STORM_CONF_DIR="" + </value> + </property> +</configuration>
