SLIDER-180. Add storm package for windows (I)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41ec7410 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41ec7410 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41ec7410 Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 41ec7410f047e28c114a77f67ac6f97763b139ad Parents: f392d23 Author: Sumit Mohanty <[email protected]> Authored: Tue Sep 30 09:05:31 2014 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Tue Sep 30 09:05:31 2014 -0700 ---------------------------------------------------------------------- app-packages/hbase-win/README.txt | 3 - app-packages/storm-win/README.txt | 36 ++ app-packages/storm-win/appConfig-default.json | 41 ++ .../storm-win/configuration/storm-env.xml | 65 +++ .../storm-win/configuration/storm-site.xml | 580 +++++++++++++++++++ app-packages/storm-win/metainfo.xml | 149 +++++ .../storm-win/package/scripts/drpc_server.py | 55 ++ .../storm-win/package/scripts/nimbus.py | 55 ++ .../storm-win/package/scripts/params.py | 56 ++ .../storm-win/package/scripts/rest_api.py | 57 ++ .../storm-win/package/scripts/service.py | 56 ++ .../storm-win/package/scripts/status_params.py | 37 ++ app-packages/storm-win/package/scripts/storm.py | 53 ++ .../storm-win/package/scripts/supervisor.py | 61 ++ .../storm-win/package/scripts/ui_server.py | 55 ++ .../storm-win/package/scripts/yaml_config.py | 80 +++ .../storm-win/package/templates/config.yaml.j2 | 28 + .../package/templates/storm_jaas.conf.j2 | 44 ++ app-packages/storm-win/pom.xml | 91 +++ app-packages/storm-win/resources-default.json | 30 + app-packages/storm-win/src/assembly/storm.xml | 68 +++ 21 files changed, 1697 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/hbase-win/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt index 0e64a84..4e0e30a 100644 --- a/app-packages/hbase-win/README.txt +++ b/app-packages/hbase-win/README.txt @@ -22,9 +22,6 @@ These files are included as reference configuration for Slider apps and are suit for a one-node cluster. -OPTION-I: Use a downloaded hbase tarball fro Windows - -****** OPTION - I ** To create the app package you will need the HBase tarball and invoke mvn command with appropriate parameters. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/README.txt ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/README.txt b/app-packages/storm-win/README.txt new file mode 100644 index 0000000..8631714 --- /dev/null +++ b/app-packages/storm-win/README.txt @@ -0,0 +1,36 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +How to create a Slider app package for Storm? + +To create the app package you will need the Storm tarball and invoke mvn command +with appropriate parameters. + +Command: +mvn clean package -Pstorm-app-package-win -Dpkg.version=<version> + -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available> + +Example: +mvn clean package -Pstorm-app-package-win -Dpkg.version=0.9.3 + -Dpkg.name=storm-0.9.3.zip -Dpkg.src=/Users/user1/Downloads + +App package can be found in + app-packages/storm-win/target/slider-storm-app-win-package-${pkg.version}.zip + +appConfig-default.json and resources-default.json are not required to be packaged. +These files are included as reference configuration for Slider apps and are suitable +for a one-node cluster. http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/appConfig-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/appConfig-default.json b/app-packages/storm-win/appConfig-default.json new file mode 100644 index 0000000..428dea5 --- /dev/null +++ b/app-packages/storm-win/appConfig-default.json @@ -0,0 +1,41 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "application.def": ".slider/package/STORM/slider-storm-app-win-package-${pkg.version}.zip", + "java_home": "C:\\java", + "create.default.zookeeper.node": "true", + + "site.global.app_user": "hadoop", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/storm-${pkg.version}", + "site.global.user_group": "hadoop", + "site.global.security_enabled": "false", + "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}", + "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}", + + "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}", + "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']", + "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}", + "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm", + "site.storm-site.transactional.zookeeper.root": "/transactional", + "site.storm-site.storm.zookeeper.port": "2181", + "site.storm-site.nimbus.childopts": "-Xmx1024m", + "site.storm-site.worker.childopts": "-Xmx768m", + "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper", + "site.storm-site.drpc.invocations.port": "0", + "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}", + "site.storm-site.transactional.zookeeper.port": "null", + "site.storm-site.nimbus.host": "${NIMBUS_HOST}", + "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}", + "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]", + "site.storm-site.supervisor.childopts": "-Xmx256m", + "site.storm-site.drpc.port": "0", + "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}" + }, + "components": { + "slider-appmaster": { + "jvm.heapsize": "256M" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/configuration/storm-env.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/configuration/storm-env.xml b/app-packages/storm-win/configuration/storm-env.xml new file mode 100644 index 0000000..091c08d --- /dev/null +++ b/app-packages/storm-win/configuration/storm-env.xml @@ -0,0 +1,65 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + + <property> + <name>kerberos_domain</name> + <value></value> + <description>The kerberos domain to be used for this Storm cluster</description> + </property> + <property> + <name>storm_client_principal_name</name> + <value></value> + <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description> + </property> + <property> + <name>storm_server_principal_name</name> + <value></value> + <description>The principal name for the Storm server to be used by Nimbus</description> + </property> + <property> + <name>storm_client_keytab</name> + <value></value> + <description>The keytab file path for Storm client</description> + </property> + <property> + <name>storm_server_keytab</name> + <value></value> + <description>The keytab file path for Storm server</description> + </property> + <!-- storm-env.sh --> + <property> + <name>content</name> + <description>This is the jinja template for storm-env.sh file</description> + <value> +#!/bin/bash + +# Set Storm specific environment variables here. + +# The java implementation to use. +export JAVA_HOME={{java_home}} + +# export STORM_CONF_DIR="" + </value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/configuration/storm-site.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/configuration/storm-site.xml b/app-packages/storm-win/configuration/storm-site.xml new file mode 100644 index 0000000..b3cce6a --- /dev/null +++ b/app-packages/storm-win/configuration/storm-site.xml @@ -0,0 +1,580 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>java.library.path</name> + <value>/usr/local/lib:/opt/local/lib:/usr/lib</value> + <description>This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers) + for the java.library.path value. java.library.path tells the JVM where + to look for native libraries. It is necessary to set this config correctly since + Storm uses the ZeroMQ and JZMQ native libs. </description> + </property> + <property> + <name>storm.local.dir</name> + <value>/hadoop/storm</value> + <description>A directory on the local filesystem used by Storm for any local + filesystem usage it needs. The directory must exist and the Storm daemons must + have permission to read/write from this location.</description> + </property> + <property> + <name>storm.zookeeper.servers</name> + <value>['localhost']</value> + <description>A list of hosts of ZooKeeper servers used to manage the cluster.</description> + </property> + <property> + <name>storm.zookeeper.port</name> + <value>2181</value> + <description>The port Storm will use to connect to each of the ZooKeeper servers.</description> + </property> + <property> + <name>storm.zookeeper.root</name> + <value>/storm</value> + <description>The root location at which Storm stores data in ZooKeeper.</description> + </property> + <property> + <name>storm.zookeeper.session.timeout</name> + <value>20000</value> + <description>The session timeout for clients to ZooKeeper.</description> + </property> + <property> + <name>storm.zookeeper.connection.timeout</name> + <value>15000</value> + <description>The connection timeout for clients to ZooKeeper.</description> + </property> + <property> + <name>storm.zookeeper.retry.times</name> + <value>5</value> + <description>The number of times to retry a Zookeeper operation.</description> + </property> + <property> + <name>storm.zookeeper.retry.interval</name> + <value>1000</value> + <description>The interval between retries of a Zookeeper operation.</description> + </property> + <property> + <name>storm.zookeeper.retry.intervalceiling.millis</name> + <value>30000</value> + <description>The ceiling of the interval between retries of a Zookeeper operation.</description> + </property> + <property> + <name>storm.cluster.mode</name> + <value>distributed</value> + <description>The mode this Storm cluster is running in. Either "distributed" or "local".</description> + </property> + <property> + <name>storm.local.mode.zmq</name> + <value>false</value> + <description>Whether or not to use ZeroMQ for messaging in local mode. If this is set + to false, then Storm will use a pure-Java messaging system. The purpose + of this flag is to make it easy to run Storm in local mode by eliminating + the need for native dependencies, which can be difficult to install. + </description> + </property> + <property> + <name>storm.thrift.transport</name> + <value>backtype.storm.security.auth.SimpleTransportPlugin</value> + <description>The transport plug-in for Thrift client/server communication.</description> + </property> + <property> + <name>storm.messaging.transport</name> + <value>backtype.storm.messaging.netty.Context</value> + <description>The transporter for communication among Storm tasks.</description> + </property> + <property> + <name>nimbus.host</name> + <value>localhost</value> + <description>The host that the master server is running on.</description> + </property> + <property> + <name>nimbus.thrift.port</name> + <value>6627</value> + <description> Which port the Thrift interface of Nimbus should run on. Clients should + connect to this port to upload jars and submit topologies.</description> + </property> + <property> + <name>nimbus.thrift.max_buffer_size</name> + <value>1048576</value> + <description>The maximum buffer size thrift should use when reading messages.</description> + </property> + <property> + <name>nimbus.childopts</name> + <value>-Xmx1024m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8649,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM</value> + <description>This parameter is used by the storm-deploy project to configure the jvm options for the nimbus daemon.</description> + </property> + <property> + <name>nimbus.task.timeout.secs</name> + <value>30</value> + <description>How long without heartbeating a task can go before nimbus will consider the task dead and reassign it to another location.</description> + </property> + <property> + <name>nimbus.supervisor.timeout.secs</name> + <value>60</value> + <description>How long before a supervisor can go without heartbeating before nimbus considers it dead and stops assigning new work to it.</description> + </property> + <property> + <name>nimbus.monitor.freq.secs</name> + <value>10</value> + <description> + How often nimbus should wake up to check heartbeats and do reassignments. Note + that if a machine ever goes down Nimbus will immediately wake up and take action. + This parameter is for checking for failures when there's no explicit event like that occuring. + </description> + </property> + <property> + <name>nimbus.cleanup.inbox.freq.secs</name> + <value>600</value> + <description>How often nimbus should wake the cleanup thread to clean the inbox.</description> + </property> + <property> + <name>nimbus.inbox.jar.expiration.secs</name> + <value>3600</value> + <description> + The length of time a jar file lives in the inbox before being deleted by the cleanup thread. + + Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS. + Note that the time it takes to delete an inbox jar file is going to be somewhat more than + NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS is set to). + </description> + </property> + <property> + <name>nimbus.task.launch.secs</name> + <value>120</value> + <description>A special timeout used when a task is initially launched. During launch, this is the timeout + used until the first heartbeat, overriding nimbus.task.timeout.secs.</description> + </property> + <property> + <name>nimbus.reassign</name> + <value>true</value> + <description>Whether or not nimbus should reassign tasks if it detects that a task goes down. + Defaults to true, and it's not recommended to change this value.</description> + </property> + <property> + <name>nimbus.file.copy.expiration.secs</name> + <value>600</value> + <description>During upload/download with the master, how long an upload or download connection is idle + before nimbus considers it dead and drops the connection.</description> + </property> + <property> + <name>nimbus.topology.validator</name> + <value>backtype.storm.nimbus.DefaultTopologyValidator</value> + <description>A custom class that implements ITopologyValidator that is run whenever a + topology is submitted. Can be used to provide business-specific logic for + whether topologies are allowed to run or not.</description> + </property> + <property> + <name>ui.port</name> + <value>8744</value> + <description>Storm UI binds to this port.</description> + </property> + <property> + <name>ui.childopts</name> + <value>-Xmx768m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf</value> + <description>Childopts for Storm UI Java process.</description> + </property> + <property> + <name>logviewer.port</name> + <value>8000</value> + <description>HTTP UI port for log viewer.</description> + </property> + <property> + <name>logviewer.childopts</name> + <value>-Xmx128m</value> + <description>Childopts for log viewer java process.</description> + </property> + <property> + <name>logviewer.appender.name</name> + <value>A1</value> + <description>Appender name used by log viewer to determine log directory.</description> + </property> + <property> + <name>drpc.port</name> + <value>3772</value> + <description>This port is used by Storm DRPC for receiving DPRC requests from clients.</description> + </property> + <property> + <name>drpc.worker.threads</name> + <value>64</value> + <description>DRPC thrift server worker threads.</description> + </property> + <property> + <name>drpc.queue.size</name> + <value>128</value> + <description>DRPC thrift server queue size.</description> + </property> + <property> + <name>drpc.invocations.port</name> + <value>3773</value> + <description>This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.</description> + </property> + <property> + <name>drpc.request.timeout.secs</name> + <value>600</value> + <description>The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also + timeout based on the socket timeout on the DRPC client, and separately based on the topology message + timeout for the topology implementing the DRPC function.</description> + </property> + <property> + <name>drpc.childopts</name> + <value>-Xmx768m</value> + <description>Childopts for Storm DRPC Java process.</description> + </property> + <property> + <name>transactional.zookeeper.root</name> + <value>/transactional</value> + <description>The root directory in ZooKeeper for metadata about TransactionalSpouts.</description> + </property> + <property> + <name>transactional.zookeeper.servers</name> + <value>null</value> + <description>The list of zookeeper servers in which to keep the transactional state. If null (which is default), + will use storm.zookeeper.servers</description> + </property> + <property> + <name>transactional.zookeeper.port</name> + <value>null</value> + <description>The port to use to connect to the transactional zookeeper servers. If null (which is default), + will use storm.zookeeper.port</description> + </property> + <property> + <name>supervisor.slots.ports</name> + <value>[6700, 6701]</value> + <description>A list of ports that can run workers on this supervisor. Each worker uses one port, and + the supervisor will only run one worker per port. Use this configuration to tune + how many workers run on each machine.</description> + </property> + <property> + <name>supervisor.childopts</name> + <value>-Xmx256m -Djava.security.auth.login.config=/etc/storm/conf/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM</value> + <description>This parameter is used by the storm-deploy project to configure the jvm options for the supervisor daemon.</description> + </property> + <property> + <name>supervisor.worker.start.timeout.secs</name> + <value>120</value> + <description>How long a worker can go without heartbeating during the initial launch before + the supervisor tries to restart the worker process. This value override + supervisor.worker.timeout.secs during launch because there is additional + overhead to starting and configuring the JVM on launch.</description> + </property> + <property> + <name>supervisor.worker.timeout.secs</name> + <value>30</value> + <description>How long a worker can go without heartbeating before the supervisor tries to restart the worker process.</description> + </property> + <property> + <name>supervisor.monitor.frequency.secs</name> + <value>3</value> + <description>How often the supervisor checks the worker heartbeats to see if any of them need to be restarted.</description> + </property> + <property> + <name>supervisor.heartbeat.frequency.secs</name> + <value>5</value> + <description>How often the supervisor sends a heartbeat to the master.</description> + </property> + <property> + <name>worker.childopts</name> + <value>-Xmx768m -javaagent:/usr/lib/storm/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=localhost,port=8650,wireformat31x=true,mode=multicast,config=/usr/lib/storm/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM</value> + <description>The jvm opts provided to workers launched by this supervisor. All \"%ID%\" substrings are replaced with an identifier for this worker.</description> + </property> + <property> + <name>worker.heartbeat.frequency.secs</name> + <value>1</value> + <description>How often this worker should heartbeat to the supervisor.</description> + </property> + <property> + <name>task.heartbeat.frequency.secs</name> + <value>3</value> + <description>How often a task should heartbeat its status to the master.</description> + </property> + <property> + <name>task.refresh.poll.secs</name> + <value>10</value> + <description>How often a task should sync its connections with other tasks (if a task is + reassigned, the other tasks sending messages to it need to refresh their connections). + In general though, when a reassignment happens other tasks will be notified + almost immediately. This configuration is here just in case that notification doesn't + come through.</description> + </property> + <property> + <name>zmq.threads</name> + <value>1</value> + <description>The number of threads that should be used by the zeromq context in each worker process.</description> + </property> + <property> + <name>zmq.linger.millis</name> + <value>5000</value> + <description>How long a connection should retry sending messages to a target host when + the connection is closed. This is an advanced configuration and can almost + certainly be ignored.</description> + </property> + <property> + <name>zmq.hwm</name> + <value>0</value> + <description>The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion + on the networking layer.</description> + </property> + <property> + <name>storm.messaging.netty.server_worker_threads</name> + <value>1</value> + <description>Netty based messaging: The # of worker threads for the server.</description> + </property> + <property> + <name>storm.messaging.netty.client_worker_threads</name> + <value>1</value> + <description>Netty based messaging: The # of worker threads for the client.</description> + </property> + <property> + <name>storm.messaging.netty.buffer_size</name> + <value>5242880</value> + <description>Netty based messaging: The buffer size for send/recv buffer.</description> + </property> + <property> + <name>storm.messaging.netty.max_retries</name> + <value>30</value> + <description>Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible.</description> + </property> + <property> + <name>storm.messaging.netty.max_wait_ms</name> + <value>1000</value> + <description>Netty based messaging: The max # of milliseconds that a peer will wait.</description> + </property> + <property> + <name>storm.messaging.netty.min_wait_ms</name> + <value>100</value> + <description>Netty based messaging: The min # of milliseconds that a peer will wait.</description> + </property> + <property> + <name>topology.enable.message.timeouts</name> + <value>true</value> + <description>True if Storm should timeout messages or not. Defaults to true. This is meant to be used + in unit tests to prevent tuples from being accidentally timed out during the test.</description> + </property> + <property> + <name>topology.debug</name> + <value>false</value> + <description>When set to true, Storm will log every message that's emitted.</description> + </property> + <property> + <name>topology.optimize</name> + <value>true</value> + <description>Whether or not the master should optimize topologies by running multiple tasks in a single thread where appropriate.</description> + </property> + <property> + <name>topology.workers</name> + <value>1</value> + <description>How many processes should be spawned around the cluster to execute this + topology. Each process will execute some number of tasks as threads within + them. This parameter should be used in conjunction with the parallelism hints + on each component in the topology to tune the performance of a topology.</description> + </property> + <property> + <name>topology.acker.executors</name> + <value>null</value> + <description>How many executors to spawn for ackers. + + If this is set to 0, then Storm will immediately ack tuples as soon + as they come off the spout, effectively disabling reliability. + </description> + </property> + <property> + <name>topology.message.timeout.secs</name> + <value>30</value> + <description>The maximum amount of time given to the topology to fully process a message + emitted by a spout. If the message is not acked within this time frame, Storm + will fail the message on the spout. Some spouts implementations will then replay + the message at a later time.</description> + </property> + <property> + <name>topology.skip.missing.kryo.registrations</name> + <value>false</value> + <description> Whether or not Storm should skip the loading of kryo registrations for which it + does not know the class or have the serializer implementation. Otherwise, the task will + fail to load and will throw an error at runtime. The use case of this is if you want to + declare your serializations on the storm.yaml files on the cluster rather than every single + time you submit a topology. Different applications may use different serializations and so + a single application may not have the code for the other serializers used by other apps. + By setting this config to true, Storm will ignore that it doesn't have those other serializations + rather than throw an error.</description> + </property> + <property> + <name>topology.max.task.parallelism</name> + <value>null</value> + <description>The maximum parallelism allowed for a component in this topology. This configuration is + typically used in testing to limit the number of threads spawned in local mode.</description> + </property> + <property> + <name>topology.max.spout.pending</name> + <value>null</value> + <description>The maximum number of tuples that can be pending on a spout task at any given time. + This config applies to individual tasks, not to spouts or topologies as a whole. + + A pending tuple is one that has been emitted from a spout but has not been acked or failed yet. + Note that this config parameter has no effect for unreliable spouts that don't tag + their tuples with a message id.</description> + </property> + <property> + <name>topology.state.synchronization.timeout.secs</name> + <value>60</value> + <description>The maximum amount of time a component gives a source of state to synchronize before it requests + synchronization again.</description> + </property> + <property> + <name>topology.stats.sample.rate</name> + <value>0.05</value> + <description>The percentage of tuples to sample to produce stats for a task.</description> + </property> + <property> + <name>topology.builtin.metrics.bucket.size.secs</name> + <value>60</value> + <description>The time period that builtin metrics data in bucketed into.</description> + </property> + <property> + <name>topology.fall.back.on.java.serialization</name> + <value>true</value> + <description>Whether or not to use Java serialization in a topology.</description> + </property> + <property> + <name>topology.worker.childopts</name> + <value>null</value> + <description>Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.</description> + </property> + <property> + <name>topology.executor.receive.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor receive queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.executor.send.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.receiver.buffer.size</name> + <value>8</value> + <description>The maximum number of messages to batch from the thread receiving off the network to the + executor queues. Must be a power of 2.</description> + </property> + <property> + <name>topology.transfer.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor transfer queue for each worker.</description> + </property> + <property> + <name>topology.tick.tuple.freq.secs</name> + <value>null</value> + <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent + to tasks. Meant to be used as a component-specific configuration.</description> + </property> + <property> + <name>topology.worker.shared.thread.pool.size</name> + <value>4</value> + <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + via the TopologyContext.</description> + </property> + <property> + <name>topology.disruptor.wait.strategy</name> + <value>com.lmax.disruptor.BlockingWaitStrategy</value> + <description>Configure the wait strategy used for internal queuing. Can be used to tradeoff latency + vs. throughput.</description> + </property> + <property> + <name>topology.executor.send.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor send queue for each executor. Must be a power of 2.</description> + </property> + <property> + <name>topology.receiver.buffer.size</name> + <value>8</value> + <description>The maximum number of messages to batch from the thread receiving off the network to the + executor queues. Must be a power of 2.</description> + </property> + <property> + <name>topology.transfer.buffer.size</name> + <value>1024</value> + <description>The size of the Disruptor transfer queue for each worker.</description> + </property> + <property> + <name>topology.tick.tuple.freq.secs</name> + <value>null</value> + <description>How often a tick tuple from the "__system" component and "__tick" stream should be sent + to tasks. Meant to be used as a component-specific configuration.</description> + </property> + <property> + <name>topology.worker.shared.thread.pool.size</name> + <value>4</value> + <description>The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed + via the TopologyContext.</description> + </property> + <property> + <name>topology.spout.wait.strategy</name> + <value>backtype.storm.spout.SleepSpoutWaitStrategy</value> + <description>A class that implements a strategy for what to do when a spout needs to wait. Waiting is + triggered in one of two conditions: + + 1. nextTuple emits no tuples + 2. The spout has hit maxSpoutPending and can't emit any more tuples</description> + </property> + <property> + <name>topology.sleep.spout.wait.strategy.time.ms</name> + <value>1</value> + <description>The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.</description> + </property> + <property> + <name>topology.error.throttle.interval.secs</name> + <value>10</value> + <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be + reported to Zookeeper per task for every 10 second interval of time.</description> + </property> + <property> + <name>topology.max.error.report.per.interval</name> + <value>5</value> + <description>The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example, + an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be + reported to Zookeeper per task for every 10 second interval of time.</description> + </property> + <property> + <name>topology.kryo.factory</name> + <value>backtype.storm.serialization.DefaultKryoFactory</value> + <description>Class that specifies how to create a Kryo instance for serialization. Storm will then apply + topology.kryo.register and topology.kryo.decorators on top of this. The default implementation + implements topology.fall.back.on.java.serialization and turns references off.</description> + </property> + <property> + <name>topology.tuple.serializer</name> + <value>backtype.storm.serialization.types.ListDelegateSerializer</value> + <description>The serializer class for ListDelegate (tuple payload). + The default serializer will be ListDelegateSerializer</description> + </property> + <property> + <name>topology.trident.batch.emit.interval.millis</name> + <value>500</value> + <description>How often a batch can be emitted in a Trident topology.</description> + </property> + <property> + <name>dev.zookeeper.path</name> + <value>/tmp/dev-storm-zookeeper</value> + <description>The path to use as the zookeeper dir when running a zookeeper server via + "storm dev-zookeeper". This zookeeper instance is only intended for development; + it is not a production grade zookeeper setup.</description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/metainfo.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/metainfo.xml b/app-packages/storm-win/metainfo.xml new file mode 100644 index 0000000..070ce35 --- /dev/null +++ b/app-packages/storm-win/metainfo.xml @@ -0,0 +1,149 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<metainfo> + <schemaVersion>2.0</schemaVersion> + <application> + <name>STORM</name> + <comment>Apache Hadoop Stream processing framework</comment> + <version>${pkg.version}</version> + <exportedConfigs>storm-site</exportedConfigs> + + <exportGroups> + <exportGroup> + <name>QuickLinks</name> + <exports> + <export> + <name>org.apache.slider.monitor</name> + <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value> + </export> + <export> + <name>nimbus.host_port</name> + <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value> + </export> + </exports> + </exportGroup> + </exportGroups> + + <commandOrders> + <commandOrder> + <command>NIMBUS-START</command> + <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED + </requires> + </commandOrder> + <commandOrder> + <command>SUPERVISOR-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>DRPC_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_UI_SERVER-START</command> + <requires>NIMBUS-STARTED</requires> + </commandOrder> + <commandOrder> + <command>STORM_REST_API-START</command> + <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires> + </commandOrder> + </commandOrders> + + <components> + + <component> + <name>NIMBUS</name> + <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <appExports>QuickLinks-nimbus.host_port</appExports> + <commandScript> + <script>scripts/nimbus.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>SUPERVISOR</name> + <category>SLAVE</category> + <autoStartOnFailure>true</autoStartOnFailure> + <componentExports> + <componentExport> + <name>log_viewer_port</name> + <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value> + </componentExport> + </componentExports> + <commandScript> + <script>scripts/supervisor.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>STORM_UI_SERVER</name> + <category>MASTER</category> + <publishConfig>true</publishConfig> + <appExports>QuickLinks-org.apache.slider.monitor</appExports> + <autoStartOnFailure>true</autoStartOnFailure> + <commandScript> + <script>scripts/ui_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>DRPC_SERVER</name> + <category>MASTER</category> + <autoStartOnFailure>true</autoStartOnFailure> + <commandScript> + <script>scripts/drpc_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osType>any</osType> + <packages> + <package> + <type>zip</type> + <name>files/${pkg.name}</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <configFiles> + <configFile> + <type>yaml</type> + <fileName>storm.yaml</fileName> + <dictionaryName>storm-site</dictionaryName> + </configFile> + <configFile> + <type>env</type> + <fileName>storm-env.sh</fileName> + <dictionaryName>storm-env</dictionaryName> + </configFile> + </configFiles> + + </application> +</metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/drpc_server.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/drpc_server.py b/app-packages/storm-win/package/scripts/drpc_server.py new file mode 100644 index 0000000..779854a --- /dev/null +++ b/app-packages/storm-win/package/scripts/drpc_server.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from storm import storm +from service import service + +class DrpcServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + storm() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + service("drpc", action="start") + + def stop(self, env): + import params + env.set_params(params) + + service("drpc", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + #check_process_status(status_params.pid_drpc) + +if __name__ == "__main__": + DrpcServer().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/nimbus.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/nimbus.py b/app-packages/storm-win/package/scripts/nimbus.py new file mode 100644 index 0000000..fafc97a --- /dev/null +++ b/app-packages/storm-win/package/scripts/nimbus.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from storm import storm +from service import service + +class Nimbus(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + storm() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + service("nimbus", action="start") + + def stop(self, env): + import params + env.set_params(params) + + service("nimbus", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + #check_process_status(status_params.pid_nimbus) + +if __name__ == "__main__": + Nimbus().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/params.py b/app-packages/storm-win/package/scripts/params.py new file mode 100644 index 0000000..c6ea62d --- /dev/null +++ b/app-packages/storm-win/package/scripts/params.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import status_params + +# server configurations +config = Script.get_config() + +app_root = config['configurations']['global']['app_root'] +conf_dir = format("{app_root}/conf") +storm_user = config['configurations']['global']['app_user'] +log_dir = config['configurations']['global']['app_log_dir'] +pid_dir = status_params.pid_dir +local_dir = config['configurations']['storm-site']['storm.local.dir'] +user_group = config['configurations']['global']['user_group'] +java64_home = config['hostLevelParams']['java_home'] +nimbus_host = config['configurations']['storm-site']['nimbus.host'] +nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] +rest_api_port = config['configurations']['global']['rest_api_port'] +rest_api_admin_port = config['configurations']['global']['rest_api_admin_port'] +rest_api_conf_file = format("{conf_dir}/config.yaml") +rest_lib_dir = format("{app_root}/external/storm-rest") +storm_bin = format("{app_root}/bin/storm.cmd") + +security_enabled = config['configurations']['global']['security_enabled'] + +if security_enabled: + _hostname_lowercase = config['hostname'].lower() + _kerberos_domain = config['configurations']['storm-env']['kerberos_domain'] + _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name'] + _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name'] + + storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase) + storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab'] + storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower()) + storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0] + storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab'] + kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/rest_api.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/rest_api.py b/app-packages/storm-win/package/scripts/rest_api.py new file mode 100644 index 0000000..6a09e4e --- /dev/null +++ b/app-packages/storm-win/package/scripts/rest_api.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from storm import storm +from service import service + + +class StormRestApi(Script): + def install(self, env): + self.install_packages(env) + self.configure(env) + + def configure(self, env): + import params + env.set_params(params) + + storm() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + service("rest_api", action="start") + + def stop(self, env): + import params + env.set_params(params) + + service("rest_api", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + #check_process_status(status_params.pid_rest_api) + +if __name__ == "__main__": + StormRestApi().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/service.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/service.py b/app-packages/storm-win/package/scripts/service.py new file mode 100644 index 0000000..dd08858 --- /dev/null +++ b/app-packages/storm-win/package/scripts/service.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + + +from resource_management import * +import time +import os +import sys + +""" +Slider package uses jps as pgrep does not list the whole process start command +""" +def service( + name, + action='start'): + import params + import status_params + + pid_file = status_params.pid_files[name] + backtype = format("backtype.storm.daemon.{name}") + + if action == "start": + cmd = format("{storm_bin} {name} > {log_dir}/{name}.out 2>&1") + + Execute(cmd, + user=params.storm_user, + logoutput=False, + wait_for_finish=False, + pid_file = pid_file + ) + + elif action == "stop": + pid = format("`cat {pid_file}` >/dev/null 2>&1") + Execute(format("kill {pid}") + ) + Execute(format("kill -9 {pid}"), + ignore_failures=True + ) + Execute(format("rm -f {pid_file}")) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/status_params.py b/app-packages/storm-win/package/scripts/status_params.py new file mode 100644 index 0000000..7dda158 --- /dev/null +++ b/app-packages/storm-win/package/scripts/status_params.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management import * + +config = Script.get_config() + +container_id = config['configurations']['global']['app_container_id'] +pid_dir = config['configurations']['global']['app_pid_dir'] +pid_nimbus = format("{pid_dir}/nimbus.pid") +pid_supervisor = format("{pid_dir}/supervisor.pid") +pid_drpc = format("{pid_dir}/drpc.pid") +pid_ui = format("{pid_dir}/ui.pid") +pid_logviewer = format("{pid_dir}/logviewer.pid") +pid_rest_api = format("{pid_dir}/restapi.pid") +pid_files = {"logviewer":pid_logviewer, + "ui": pid_ui, + "nimbus": pid_nimbus, + "supervisor": pid_supervisor, + "rest_api": pid_rest_api, + "drpc": pid_drpc} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/storm.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/storm.py b/app-packages/storm-win/package/scripts/storm.py new file mode 100644 index 0000000..e109826 --- /dev/null +++ b/app-packages/storm-win/package/scripts/storm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from yaml_config import yaml_config +import sys + +def storm(): + import params + + Directory([params.log_dir, params.pid_dir, params.local_dir, params.conf_dir], + owner=params.storm_user, + group=params.user_group, + recursive=True + ) + + File(format("{conf_dir}/config.yaml"), + content=Template("config.yaml.j2"), + owner = params.storm_user, + group = params.user_group + ) + + yaml_config( "storm.yaml", + conf_dir = params.conf_dir, + configurations = params.config['configurations']['storm-site'], + owner = params.storm_user, + group = params.user_group + ) + + if params.security_enabled: + File(format("{conf_dir}/storm_jaas.conf"), + content=Template("storm_jaas.conf.j2"), + owner = params.storm_user, + group = params.user_group + ) + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/supervisor.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/supervisor.py b/app-packages/storm-win/package/scripts/supervisor.py new file mode 100644 index 0000000..eba9fa9 --- /dev/null +++ b/app-packages/storm-win/package/scripts/supervisor.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from yaml_config import yaml_config +from storm import storm +from service import service + + +class Supervisor(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + storm() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + service("supervisor", action="start") + service("logviewer", action="start") + + def stop(self, env): + import params + env.set_params(params) + + service("supervisor", action="stop") + service("logviewer", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + + #check_process_status(status_params.pid_supervisor) + + +if __name__ == "__main__": + Supervisor().execute() + http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/ui_server.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/ui_server.py b/app-packages/storm-win/package/scripts/ui_server.py new file mode 100644 index 0000000..0875b76 --- /dev/null +++ b/app-packages/storm-win/package/scripts/ui_server.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from storm import storm +from service import service + +class UiServer(Script): + def install(self, env): + self.install_packages(env) + + def configure(self, env): + import params + env.set_params(params) + + storm() + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + + service("ui", action="start") + + def stop(self, env): + import params + env.set_params(params) + + service("ui", action="stop") + + def status(self, env): + import status_params + env.set_params(status_params) + #check_process_status(status_params.pid_ui) + +if __name__ == "__main__": + UiServer().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/scripts/yaml_config.py ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/scripts/yaml_config.py b/app-packages/storm-win/package/scripts/yaml_config.py new file mode 100644 index 0000000..5f763cc --- /dev/null +++ b/app-packages/storm-win/package/scripts/yaml_config.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import re +import socket +from resource_management import * + +def escape_yaml_propetry(value): + # pre-process value for any "_HOST" tokens + value = value.replace('_HOST', socket.getfqdn()) + + unquouted = False + unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"] + + if value in unquouted_values: + unquouted = True + + # if is list [a,b,c] + if re.match('^\w*\[.+\]\w*$', value): + unquouted = True + + # if is map {'a':'b'} + if re.match('^\w*\{.+\}\w*$', value): + unquouted = True + + try: + int(value) + unquouted = True + except ValueError: + pass + + try: + float(value) + unquouted = True + except ValueError: + pass + + if not unquouted: + value = value.replace("'","''") + value = "'"+value+"'" + + return value + +def yaml_inline_template(configurations): + return source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }} +{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry]) + +def yaml_config( + filename, + configurations = None, + conf_dir = None, + mode = None, + owner = None, + group = None +): + config_content = yaml_inline_template(configurations) + + File (format("{conf_dir}/{filename}"), + content = config_content, + owner = owner, + group = group, + mode = mode + ) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/templates/config.yaml.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/templates/config.yaml.j2 b/app-packages/storm-win/package/templates/config.yaml.j2 new file mode 100644 index 0000000..58f5c9e --- /dev/null +++ b/app-packages/storm-win/package/templates/config.yaml.j2 @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nimbusHost: {{nimbus_host}} +nimbusPort: {{nimbus_port}} + +# HTTP-specific options. +http: + + # The port on which the HTTP server listens for service requests. + port: {{rest_api_port}} + + # The port on which the HTTP server listens for administrative requests. + adminPort: {{rest_api_admin_port}} + +enableGanglia: false \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/package/templates/storm_jaas.conf.j2 ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/package/templates/storm_jaas.conf.j2 b/app-packages/storm-win/package/templates/storm_jaas.conf.j2 new file mode 100644 index 0000000..a1ba6ea --- /dev/null +++ b/app-packages/storm-win/package/templates/storm_jaas.conf.j2 @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +StormServer { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_server_keytab_path}}" + storeKey=true + useTicketCache=false + principal="{{storm_jaas_server_principal}}"; +}; +StormClient { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_client_keytab_path}}" + storeKey=true + useTicketCache=false + serviceName="{{storm_jaas_stormclient_servicename}}" + debug=true + principal="{{storm_jaas_client_principal}}"; +}; +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + keyTab="{{storm_client_keytab_path}}" + storeKey=true + useTicketCache=false + serviceName="zookeeper" + principal="{{storm_jaas_client_principal}}"; +}; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/pom.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/pom.xml b/app-packages/storm-win/pom.xml new file mode 100644 index 0000000..490ca96 --- /dev/null +++ b/app-packages/storm-win/pom.xml @@ -0,0 +1,91 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <parent> + <groupId>org.apache.slider</groupId> + <artifactId>slider</artifactId> + <version>0.51.0-incubating-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>slider-storm-app-win-package</artifactId> + <packaging>pom</packaging> + <name>Slider Storm App Package</name> + <description>Slider Storm App Package</description> + <version>${pkg.version}</version> + <properties> + <work.dir>package-tmp</work.dir> + </properties> + + <profiles> + <profile> + <id>storm-app-package-win</id> + <build> + <plugins> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>copy</id> + <phase>validate</phase> + <configuration> + <target name="copy and rename file"> + <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptor>src/assembly/storm.xml</descriptor> + <appendAssemblyId>false</appendAssemblyId> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + + </plugins> + </build> + </profile> + </profiles> + + <build> + </build> + + <dependencies> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/resources-default.json ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/resources-default.json b/app-packages/storm-win/resources-default.json new file mode 100644 index 0000000..31b445e --- /dev/null +++ b/app-packages/storm-win/resources-default.json @@ -0,0 +1,30 @@ +{ + "schema" : "http://example.org/specification/v2.0.0";, + "metadata" : { + }, + "global" : { + "yarn.log.include.patterns": "", + "yarn.log.exclude.patterns": "", + "yarn.log.interval": "0" + }, + "components": { + "slider-appmaster": { + }, + "NIMBUS": { + "yarn.role.priority": "1", + "yarn.component.instances": "1" + }, + "STORM_UI_SERVER": { + "yarn.role.priority": "2", + "yarn.component.instances": "1" + }, + "DRPC_SERVER": { + "yarn.role.priority": "3", + "yarn.component.instances": "1" + }, + "SUPERVISOR": { + "yarn.role.priority": "4", + "yarn.component.instances": "1" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41ec7410/app-packages/storm-win/src/assembly/storm.xml ---------------------------------------------------------------------- diff --git a/app-packages/storm-win/src/assembly/storm.xml b/app-packages/storm-win/src/assembly/storm.xml new file mode 100644 index 0000000..2ee7d31 --- /dev/null +++ b/app-packages/storm-win/src/assembly/storm.xml @@ -0,0 +1,68 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + <id>storm_v${storm.version}</id> + <formats> + <format>zip</format> + <format>dir</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <files> + <file> + <source>appConfig-default.json</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>metainfo.xml</source> + <outputDirectory>/</outputDirectory> + <filtered>true</filtered> + <fileMode>0755</fileMode> + </file> + <file> + <source>${pkg.src}/${pkg.name}</source> + <outputDirectory>package/files</outputDirectory> + <filtered>false</filtered> + <fileMode>0755</fileMode> + </file> + </files> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <outputDirectory>/</outputDirectory> + <excludes> + <exclude>pom.xml</exclude> + <exclude>src/**</exclude> + <exclude>target/**</exclude> + <exclude>appConfig-default.json</exclude> + <exclude>metainfo.xml</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + </fileSets> +</assembly>
