Repository: falcon Updated Branches: refs/heads/master f982f86c7 -> 2bf90130d
http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/scheduler/src/test/resources/startup.properties ---------------------------------------------------------------------- diff --git a/scheduler/src/test/resources/startup.properties b/scheduler/src/test/resources/startup.properties new file mode 100644 index 0000000..2e938ee --- /dev/null +++ b/scheduler/src/test/resources/startup.properties @@ -0,0 +1,154 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +*.domain=debug + +######### Implementation classes ######### +## DONT MODIFY UNLESS SURE ABOUT CHANGE ## + +*.workflow.engine.impl=org.apache.falcon.workflow.engine.OozieWorkflowEngine +*.oozie.process.workflow.builder=org.apache.falcon.workflow.OozieProcessWorkflowBuilder +*.oozie.feed.workflow.builder=org.apache.falcon.workflow.OozieFeedWorkflowBuilder +*.SchedulableEntityManager.impl=org.apache.falcon.resource.SchedulableEntityManager +*.ConfigSyncService.impl=org.apache.falcon.resource.ConfigSyncService +*.ProcessInstanceManager.impl=org.apache.falcon.resource.InstanceManager +*.catalog.service.impl=org.apache.falcon.catalog.HiveCatalogService + +##### Falcon Services ##### +*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.notification.service.impl.JobCompletionService,\ + org.apache.falcon.notification.service.impl.SchedulerService,\ + org.apache.falcon.notification.service.impl.AlarmService,\ + org.apache.falcon.notification.service.impl.DataAvailabilityService,\ + org.apache.falcon.execution.FalconExecutionService,\ + org.apache.falcon.state.store.service.FalconJPAService + +##### Falcon Configuration Store Change listeners ##### +*.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ + org.apache.falcon.entity.ColoClusterRelation,\ + org.apache.falcon.group.FeedGroupMap,\ + org.apache.falcon.entity.store.FeedLocationStore + +##### JMS MQ Broker Implementation class ##### +*.broker.impl.class=org.apache.activemq.ActiveMQConnectionFactory + + +######### System startup parameters ######### + +# Location to store user entity configurations +debug.config.store.uri=file://${user.dir}/target/store +debug.config.store.persist=false +debug.config.oozie.conf.uri=${user.dir}/target/oozie +debug.system.lib.location=${system.lib.location} +debug.broker.url=vm://localhost +debug.retry.recorder.path=${user.dir}/target/retry +debug.libext.feed.retention.paths=${falcon.libext} +debug.libext.feed.replication.paths=${falcon.libext} +debug.libext.process.paths=${falcon.libext} + +*.falcon.cleanup.service.frequency=minutes(5) + + +######### Properties for configuring JMS provider - activemq ######### +# Default Active MQ url +*.broker.url=tcp://localhost:61616 + +# default time-to-live for a JMS message 3 days (time in minutes) +*.broker.ttlInMins=4320 +*.entity.topic=FALCON.ENTITY.TOPIC +*.max.retry.failure.count=1 +*.retry.recorder.path=${user.dir}/logs/retry + +######### Properties for configuring iMon client and metric ######### +*.internal.queue.size=1000 + + +##### List of shared libraries for Falcon workflows ##### +*.shared.libs=activemq-all,ant,geronimo-j2ee-management,jms,json-simple,oozie-client,spring-jms,commons-lang3 + +######### Authentication Properties ######### + +# Authentication type must be specified: simple|kerberos +*.falcon.authentication.type=simple + +##### Service Configuration + +# Indicates the Kerberos principal to be used in Falcon Service. +*.falcon.service.authentication.kerberos.principal= + +# Location of the keytab file with the credentials for the Service principal. +*.falcon.service.authentication.kerberos.keytab= + +# name node principal to talk to config store +*.dfs.namenode.kerberos.principal= + +##### SPNEGO Configuration + +# Authentication type must be specified: simple|kerberos|<class> +# org.apache.falcon.security.RemoteUserInHeaderBasedAuthenticationHandler can be used for backwards compatibility +*.falcon.http.authentication.type=simple + +# Indicates how long (in seconds) an authentication token is valid before it has to be renewed. +*.falcon.http.authentication.token.validity=36000 + +# The signature secret for signing the authentication tokens. +*.falcon.http.authentication.signature.secret=falcon + +# The domain to use for the HTTP cookie that stores the authentication token. +*.falcon.http.authentication.cookie.domain= + +# Indicates if anonymous requests are allowed when using 'simple' authentication. +*.falcon.http.authentication.simple.anonymous.allowed=false + +# Indicates the Kerberos principal to be used for HTTP endpoint. +# The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. +*.falcon.http.authentication.kerberos.principal= + +# Location of the keytab file with the credentials for the HTTP principal. +*.falcon.http.authentication.kerberos.keytab= + +# The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's KerberosName for more details. +*.falcon.http.authentication.kerberos.name.rules=DEFAULT + +# Comma separated list of black listed users +*.falcon.http.authentication.blacklisted.users= + + +######## StateStore Properties ##### +*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore +*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +*.falcon.statestore.jdbc.url=jdbc:derby:target/test-data/data.db;create=true +*.falcon.statestore.jdbc.username=sa +*.falcon.statestore.jdbc.password= +*.falcon.statestore.connection.data.source=org.apache.commons.dbcp.BasicDataSource +# Maximum number of active connections that can be allocated from this pool at the same time. +*.falcon.statestore.pool.max.active.conn=10 +*.falcon.statestore.connection.properties= +# Indicates the interval (in milliseconds) between eviction runs. +*.falcon.statestore.validate.db.connection.eviction.interval=300000 +# The number of objects to examine during each run of the idle object evictor thread. +*.falcon.statestore.validate.db.connection.eviction.num=10 +# Creates Falcon DB. +# If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. +# If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. +*.falcon.statestore.create.db.schema=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/src/bin/falcon-db.sh ---------------------------------------------------------------------- diff --git a/src/bin/falcon-db.sh b/src/bin/falcon-db.sh new file mode 100644 index 0000000..415fd5d --- /dev/null +++ b/src/bin/falcon-db.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +# + +# resolve links - $0 may be a softlink +PRG="${0}" + +while [ -h "${PRG}" ]; do + ls=`ls -ld "${PRG}"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "${PRG}"`/"$link" + fi +done + +BASEDIR=`dirname ${PRG}` +BASEDIR=`cd ${BASEDIR}/..;pwd` + +. ${BASEDIR}/bin/falcon-config.sh 'server' falcon + +if test -z ${JAVA_HOME} +then + JAVA_BIN=java +else + JAVA_BIN=${JAVA_HOME}/bin/java +fi + +while [[ ${1} =~ ^\-D ]]; do + JAVA_PROPERTIES="${JAVA_PROPERTIES} ${1}" + shift +done + +# set the client class path +FALCONCPPATH="$FALCONCPPATH:${BASEDIR}/client/lib/*" + +${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${FALCONCPPATH} org.apache.falcon.tools.FalconStateStoreDBCLI "${@}" http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index ce6e91f..5ddba01 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -57,7 +57,8 @@ # org.apache.falcon.notification.service.impl.SchedulerService,\ # org.apache.falcon.notification.service.impl.AlarmService,\ # org.apache.falcon.notification.service.impl.DataAvailabilityService,\ -# org.apache.falcon.execution.FalconExecutionService +# org.apache.falcon.execution.FalconExecutionService,\ +# org.apache.falcon.state.store.service.FalconJPAService ##### Prism Services ##### prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\ @@ -262,3 +263,22 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ # Setting monitoring plugin, if SMTP parameters is defined #*.monitoring.plugins=org.apache.falcon.plugin.DefaultMonitoringPlugin,\ # org.apache.falcon.plugin.EmailNotificationPlugin + +######### StateStore Properties ##### +#*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore +#*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +#*.falcon.statestore.jdbc.url=jdbc:derby:data/statestore.db;create=true +#*.falcon.statestore.jdbc.username=sa +#*.falcon.statestore.jdbc.password= +#*.falcon.statestore.connection.data.source=org.apache.commons.dbcp.BasicDataSource +## Maximum number of active connections that can be allocated from this pool at the same time. +#*.falcon.statestore.pool.max.active.conn=10 +#*.falcon.statestore.connection.properties= +## Indicates the interval (in milliseconds) between eviction runs. +#*.falcon.statestore.validate.db.connection.eviction.interval=300000 +## The number of objects to examine during each run of the idle object evictor thread. +#*.falcon.statestore.validate.db.connection.eviction.num=10 +## Creates Falcon DB. +## If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. +## If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. +#*.falcon.statestore.create.db.schema=true http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/src/main/assemblies/distributed-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml index 794eaef..ebd1745 100644 --- a/src/main/assemblies/distributed-package.xml +++ b/src/main/assemblies/distributed-package.xml @@ -41,6 +41,11 @@ </fileSet> <fileSet> + <directory>scheduler/target/dependency</directory> + <outputDirectory>client/lib</outputDirectory> + </fileSet> + + <fileSet> <directory>oozie-el-extensions/target/dependency</directory> <outputDirectory>oozie/libext</outputDirectory> </fileSet> http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml index fcff8d7..b88aec3 100644 --- a/src/main/assemblies/standalone-package.xml +++ b/src/main/assemblies/standalone-package.xml @@ -40,6 +40,11 @@ </fileSet> <fileSet> + <directory>scheduler/target/dependency</directory> + <outputDirectory>client/lib</outputDirectory> + </fileSet> + + <fileSet> <directory>oozie-el-extensions/target/dependency</directory> <outputDirectory>oozie/libext</outputDirectory> </fileSet> http://git-wip-us.apache.org/repos/asf/falcon/blob/6d313855/unit/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/unit/src/main/resources/startup.properties b/unit/src/main/resources/startup.properties index fe6f430..4576e0b 100644 --- a/unit/src/main/resources/startup.properties +++ b/unit/src/main/resources/startup.properties @@ -127,3 +127,21 @@ debug.libext.process.paths=${falcon.libext} # Comma separated list of black listed users *.falcon.http.authentication.blacklisted.users= +######### StateStore Properties ##### +#*.falcon.state.store.impl=org.apache.falcon.state.store.jdbc.JDBCStateStore +#*.falcon.statestore.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver +#*.falcon.statestore.jdbc.url=jdbc:derby:data/statestore.db;create=true +#*.falcon.statestore.jdbc.username=sa +#*.falcon.statestore.jdbc.password= +#*.falcon.statestore.connection.data.source=org.apache.commons.dbcp.BasicDataSource +## Maximum number of active connections that can be allocated from this pool at the same time. +#*.falcon.statestore.pool.max.active.conn=10 +#*.falcon.statestore.connection.properties= +## Indicates the interval (in milliseconds) between eviction runs. +#*.falcon.statestore.validate.db.connection.eviction.interval=300000 +## The number of objects to examine during each run of the idle object evictor thread. +#*.falcon.statestore.validate.db.connection.eviction.num=10 +## Creates Falcon DB. +## If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. +## If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. +#*.falcon.statestore.create.db.schema=true
