Repository: falcon Updated Branches: refs/heads/master 123391d46 -> 4ab28e80f
89 Pull request is available that will extend the cluster and process XSD to include the spark execution engine. Author: peeyush b <[email protected]> Reviewers: Pallavi <[email protected]>, Pavan <[email protected]>, Venkat <[email protected]>, Venkatesan <[email protected]> Closes #89 from peeyushb/FALCON-1817 and squashes the following commits: 64d8c35 [peeyush b] making name as required parameter b26af4e [peeyush b] removed line 8056916 [peeyush b] Incorporating review comments Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4ab28e80 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4ab28e80 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4ab28e80 Branch: refs/heads/master Commit: 4ab28e80f908ad8f3aac65c2dd39eb7f49b213ae Parents: 123391d Author: Peeyush Bishnoi <[email protected]> Authored: Wed Apr 27 21:32:27 2016 +0530 Committer: peeyush b <[email protected]> Committed: Wed Apr 27 21:32:27 2016 +0530 ---------------------------------------------------------------------- client/src/main/resources/cluster-0.1.xsd | 2 ++ client/src/main/resources/process-0.1.xsd | 32 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/4ab28e80/client/src/main/resources/cluster-0.1.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/cluster-0.1.xsd b/client/src/main/resources/cluster-0.1.xsd index 34e3689..0e0ada8 100644 --- a/client/src/main/resources/cluster-0.1.xsd +++ b/client/src/main/resources/cluster-0.1.xsd @@ -167,6 +167,7 @@ messaging specifies the interface for sending feed availability messages, it's endpoint is broker url with tcp address.ex: tcp://corp.messaging:61616?daemon=true registry specifies the interface for Hcatalog. + spark specifies the interface for spark master. </xs:documentation> </xs:annotation> <xs:restriction base="xs:string"> @@ -176,6 +177,7 @@ <xs:enumeration value="workflow"/> <xs:enumeration value="messaging"/> <xs:enumeration value="registry"/> + <xs:enumeration value="spark"/> </xs:restriction> </xs:simpleType> <xs:simpleType name="IDENTIFIER"> http://git-wip-us.apache.org/repos/asf/falcon/blob/4ab28e80/client/src/main/resources/process-0.1.xsd ---------------------------------------------------------------------- diff --git a/client/src/main/resources/process-0.1.xsd b/client/src/main/resources/process-0.1.xsd index 9d7898f..456ebf9 100644 --- a/client/src/main/resources/process-0.1.xsd +++ b/client/src/main/resources/process-0.1.xsd @@ -152,6 +152,13 @@ </xs:documentation> </xs:annotation> </xs:element> + <xs:element type="spark-attributes" name="spark-attributes" minOccurs="0"> + <xs:annotation> + <xs:documentation>Defines the spark related attributes to run the Spark application. Attributes + must be defined when workflow engine "spark" is selected. + </xs:documentation> + </xs:annotation> + </xs:element> <xs:element type="retry" name="retry" minOccurs="0"> <xs:annotation> <xs:documentation>Retry defines how to handle workflow failures. Policy type - periodic, exponential @@ -331,9 +338,34 @@ <xs:enumeration value="oozie"/> <xs:enumeration value="pig"/> <xs:enumeration value="hive"/> + <xs:enumeration value="spark"/> </xs:restriction> </xs:simpleType> + <xs:complexType name="spark-attributes"> + <xs:annotation> + <xs:documentation> + List of Spark attributes to run Spark application: + master element is optional and indicates the url of the Spark Master. Ex: yarn-client, yarn-master, or local. This will override the Spark interface, if define in cluster entity. + mode element if present indicates the mode of spark, where to run spark driver program. Ex: client,cluster. + name element indicates the name of the spark application. + class element if present, indicates the spark's application main class. + jar element indicates a comma separated list of jars or python files. Also provide the jar or file directory path to workflow attribute path + spark-opts element if present, contains a list of spark options that can be passed to spark driver. + arg element if present, contains arguments that can be passed to spark application. + </xs:documentation> + </xs:annotation> + <xs:sequence> + <xs:element name="master" type="xs:string" minOccurs="0" maxOccurs="1"/> + <xs:element name="mode" type="xs:string" minOccurs="0" maxOccurs="1"/> + <xs:element name="name" type="xs:string" minOccurs="1" maxOccurs="1"/> + <xs:element name="class" type="xs:string" minOccurs="0" maxOccurs="1"/> + <xs:element name="jar" type="xs:string" minOccurs="1" maxOccurs="1"/> + <xs:element name="spark-opts" type="xs:string" minOccurs="0" maxOccurs="1"/> + <xs:element name="arg" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> + </xs:sequence> + </xs:complexType> + <xs:complexType name="retry"> <xs:attribute type="policy-type" name="policy" use="required"/> <xs:attribute type="frequency-type" name="delay" use="required"/>
