Repository: falcon Updated Branches: refs/heads/master 85345ad7e -> a83a141d6
FALCON-1936 Extensions related files are not available in distro packaging Author: Sowmya Ramesh <[email protected]> Reviewers: "Venkat Ranganathan <[email protected]>" Closes #125 from sowmyaramesh/FALCON-1936_1 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/a83a141d Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/a83a141d Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/a83a141d Branch: refs/heads/master Commit: a83a141d60de60d97b0fd4a81c3fadce880ed308 Parents: 85345ad Author: Sowmya Ramesh <[email protected]> Authored: Thu May 5 11:25:30 2016 -0700 Committer: Sowmya Ramesh <[email protected]> Committed: Thu May 5 11:25:30 2016 -0700 ---------------------------------------------------------------------- common/src/main/resources/startup.properties | 3 +- docs/src/site/twiki/Extensions.twiki | 12 ++++- src/conf/startup.properties | 6 ++- src/main/assemblies/assembly-standalone.xml | 46 +++++++++---------- src/main/assemblies/distributed-package.xml | 54 +++++++++++++++++----- src/main/assemblies/standalone-package.xml | 56 +++++++++++++++++------ 6 files changed, 123 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 7a850f8..37c09e0 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -43,7 +43,8 @@ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ org.apache.falcon.service.ProxyUserService,\ - org.apache.falcon.service.FalconJPAService + org.apache.falcon.service.FalconJPAService,\ + org.apache.falcon.extensions.ExtensionService ## Add if you want to use Falcon Azure integration ## # org.apache.falcon.adfservice.ADFProviderService ## If you wish to use Falcon native scheduler add the commented out services below to application.services ## http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/docs/src/site/twiki/Extensions.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/Extensions.twiki b/docs/src/site/twiki/Extensions.twiki index 6b4bf11..8e74321 100644 --- a/docs/src/site/twiki/Extensions.twiki +++ b/docs/src/site/twiki/Extensions.twiki @@ -46,10 +46,18 @@ Sample extensions are published in addons/extensions ---++ Packaging and installation -Extension artifacts in addons/extensions are packaged in falcon war under extensions directory. For manual installation user is expected to install the extension artifacts under extensions in falcon war to HDFS at "extension.store.uri" path defined in startup properties and then restart Falcon. +This feature is enabled by default but could be disabled by removing the following from startup properties: +<verbatim> +config name: *.application.services +config value: org.apache.falcon.extensions.ExtensionService +</verbatim> + +For manual installation user is expected to update "extension.store.uri" property defined in startup properties with +HDFS path where the extension artifacts will be copied to. +Extension artifacts in addons/extensions are packaged in falcon. For manual installation once the Falcon Server is setup user is expected to copy the extension artifacts under {falcon-server-dir}/extensions to HDFS at "extension.store.uri" path defined in startup properties and then restart Falcon. ---++ Migration Recipes framework and HDFS mirroring capability was added in Apache Falcon 0.6.0 release and it was client side logic. With 0.10 release its moved to server side and renamed as server side extensions. Client side recipes only had CLI support and expected certain pre steps to get it working. This is no longer required in 0.10 release as new CLI and REST API support has been provided. If user is migrating to 0.10 release and above then old Recipe setup and CLI's won't work. For manual installation user is expected to copy Extension artifacts to HDFS. Please refer "Packaging and installation" section above for more details. -Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's for extension jobs and instances management. \ No newline at end of file +Please Refer to [[falconcli/FalconCLI][Falcon CLI]] and [[restapi/ResourceList][REST API]] for more details on usage of CLI and REST API's for extension jobs and instances management. http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index ad322c7..47895dd 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -51,7 +51,8 @@ org.apache.falcon.metadata.MetadataMappingService,\ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ - org.apache.falcon.service.ProxyUserService + org.apache.falcon.service.ProxyUserService,\ + org.apache.falcon.extensions.ExtensionService ## Add if you want to use Falcon Azure integration ## # org.apache.falcon.adfservice.ADFProviderService ## If you wish to use Falcon native scheduler uncomment out below application services and comment out above application services ## @@ -116,6 +117,9 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ # Location to store user entity configurations *.config.store.uri=file://${falcon.home}/data/${falcon.app.type}-store +# HDFS location of extension store for extension artifacts +*.extension.store.uri= + #Location to store state of Feed SLA monitoring service *.feed.sla.service.store.uri = file://${falcon.home}/data/sla/pendingfeedinstances http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/src/main/assemblies/assembly-standalone.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/assembly-standalone.xml b/src/main/assemblies/assembly-standalone.xml index b9d705e..b356484 100644 --- a/src/main/assemblies/assembly-standalone.xml +++ b/src/main/assemblies/assembly-standalone.xml @@ -138,25 +138,8 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> - <outputDirectory>extensions/hdfs-snapshot-mirroring/META</outputDirectory> - <fileMode>0755</fileMode> - <directoryMode>0755</directoryMode> - </fileSet> - - <fileSet> - <directory>./</directory> - <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/build</outputDirectory> - <excludes> - <exclude>*/**</exclude> - </excludes> - <fileMode>0755</fileMode> - <directoryMode>0755</directoryMode> - </fileSet> - - <fileSet> <directory>./</directory> - <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/runtime</outputDirectory> + <outputDirectory>extensions/hdfs-mirroring/resources/build</outputDirectory> <excludes> <exclude>*/**</exclude> </excludes> @@ -165,13 +148,6 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/resources</directory> - <outputDirectory>extensions/hdfs-snapshot-mirroring/resources</outputDirectory> - <fileMode>0755</fileMode> - <directoryMode>0755</directoryMode> - </fileSet> - - <fileSet> <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/META</outputDirectory> <fileMode>0755</fileMode> @@ -206,6 +182,16 @@ </fileSet> <fileSet> + <directory>./</directory> + <outputDirectory>extensions/hdfs-snapshot-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> <directory>addons/extensions/hive-mirroring/src/main/META</directory> <outputDirectory>extensions/hive-mirroring/META</outputDirectory> <fileMode>0755</fileMode> @@ -238,6 +224,16 @@ <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> + + <fileSet> + <directory>./</directory> + <outputDirectory>extensions/hive-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> </fileSets> <files> http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/src/main/assemblies/distributed-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/distributed-package.xml b/src/main/assemblies/distributed-package.xml index a5171c4..1b5353d 100644 --- a/src/main/assemblies/distributed-package.xml +++ b/src/main/assemblies/distributed-package.xml @@ -116,14 +116,14 @@ <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>addons/extensions/hdfs-mirroring/src/main/META</directory> + <directory>../addons/extensions/hdfs-mirroring/src/main/META</directory> <outputDirectory>extensions/hdfs-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-mirroring/libs/build</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -133,7 +133,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -143,21 +143,31 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hdfs-mirroring/src/main/resources</directory> <outputDirectory>extensions/hdfs-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> + <directory>../</directory> + <outputDirectory>extensions/hdfs-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/build</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -167,7 +177,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -177,21 +187,31 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hdfs-snapshot-mirroring/src/main/resources</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>addons/extensions/hive-mirroring/src/main/META</directory> + <directory>../</directory> + <outputDirectory>extensions/hdfs-snapshot-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../addons/extensions/hive-mirroring/src/main/META</directory> <outputDirectory>extensions/hive-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <fileMode>0755</fileMode> <outputDirectory>extensions/hive-mirroring/libs/build</outputDirectory> <excludes> @@ -201,7 +221,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hive-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -211,13 +231,23 @@ </fileSet> <fileSet> - <directory>addons/extensions/hive-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hive-mirroring/src/main/resources</directory> <outputDirectory>extensions/hive-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> + <directory>../</directory> + <outputDirectory>extensions/hive-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> <directory>../examples/app</directory> <outputDirectory>examples/app</outputDirectory> <directoryMode>0755</directoryMode> http://git-wip-us.apache.org/repos/asf/falcon/blob/a83a141d/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/standalone-package.xml b/src/main/assemblies/standalone-package.xml index b5db01c..e97eceb 100644 --- a/src/main/assemblies/standalone-package.xml +++ b/src/main/assemblies/standalone-package.xml @@ -87,7 +87,7 @@ </fileSet> <fileSet> - <directory>.././</directory> + <directory>../</directory> <outputDirectory>/</outputDirectory> <fileMode>0644</fileMode> <includes> @@ -133,14 +133,14 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-mirroring/src/main/META</directory> + <directory>../addons/extensions/hdfs-mirroring/src/main/META</directory> <outputDirectory>extensions/hdfs-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-mirroring/libs/build</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -150,7 +150,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -160,21 +160,31 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hdfs-mirroring/src/main/resources</directory> <outputDirectory>extensions/hdfs-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> + <directory>../</directory> + <outputDirectory>extensions/hdfs-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../addons/extensions/hdfs-snapshot-mirroring/src/main/META</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/build</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -184,7 +194,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -194,21 +204,31 @@ </fileSet> <fileSet> - <directory>addons/extensions/hdfs-snapshot-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hdfs-snapshot-mirroring/src/main/resources</directory> <outputDirectory>extensions/hdfs-snapshot-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>addons/extensions/hive-mirroring/src/main/META</directory> + <directory>../</directory> + <outputDirectory>extensions/hdfs-snapshot-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> + + <fileSet> + <directory>../addons/extensions/hive-mirroring/src/main/META</directory> <outputDirectory>extensions/hive-mirroring/META</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <fileMode>0755</fileMode> <outputDirectory>extensions/hive-mirroring/libs/build</outputDirectory> <excludes> @@ -218,7 +238,7 @@ </fileSet> <fileSet> - <directory>./</directory> + <directory>../</directory> <outputDirectory>extensions/hive-mirroring/libs/runtime</outputDirectory> <excludes> <exclude>*/**</exclude> @@ -228,11 +248,21 @@ </fileSet> <fileSet> - <directory>addons/extensions/hive-mirroring/src/main/resources</directory> + <directory>../addons/extensions/hive-mirroring/src/main/resources</directory> <outputDirectory>extensions/hive-mirroring/resources</outputDirectory> <fileMode>0755</fileMode> <directoryMode>0755</directoryMode> </fileSet> + + <fileSet> + <directory>../</directory> + <outputDirectory>extensions/hive-mirroring/resources/build</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + </fileSet> </fileSets> <files>
