SPOI-8656 filter example app Incorporating review comments
1. Updating README 2. added log4j properties 3. Updated test cases. Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/0590c258 Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/0590c258 Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/0590c258 Branch: refs/heads/master Commit: 0590c25820b89b5616156587c0df107a6e871c2a Parents: f7c7b7c Author: yogidevendra <[email protected]> Authored: Mon Jul 18 11:55:22 2016 +0530 Committer: Lakshmi Prasanna Velineni <[email protected]> Committed: Sun Mar 26 11:43:48 2017 -0700 ---------------------------------------------------------------------- examples/filter/README.md | 20 + examples/filter/XmlJavadocCommentsExtractor.xsl | 44 ++ examples/filter/pom.xml | 312 +++++++++++++++ examples/filter/src/assemble/appPackage.xml | 43 ++ .../tutorial/filter/Application.java | 49 +++ .../tutorial/filter/TransactionPOJO.java | 64 +++ .../src/main/resources/META-INF/input.txt | 401 +++++++++++++++++++ .../src/main/resources/META-INF/properties.xml | 95 +++++ .../main/resources/META-INF/rejected_output.txt | 326 +++++++++++++++ .../main/resources/META-INF/selected_output.txt | 74 ++++ .../tutorial/filter/ApplicationTest.java | 111 +++++ .../filter/src/test/resources/log4j.properties | 22 + 12 files changed, 1561 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/README.md ---------------------------------------------------------------------- diff --git a/examples/filter/README.md b/examples/filter/README.md new file mode 100644 index 0000000..367495a --- /dev/null +++ b/examples/filter/README.md @@ -0,0 +1,20 @@ +# Filter operator example + +Sample application to show how to use the filter operator. + +The application reads transaction records from a csv file using `FSRecordReaderModule`. Then converts these records into plain old java objects (POJO) using CSVParser. These POJOs are filtered based on given condition. POJOs meeting the filter criteria are written to `selected.txt`. POJOs not meeting filter criteria are written to `rejected.txt`. Writing POJO output to file is done using `CSVFormatter` and `StringFileOutputOperator` + +### How to configure +The properties file META-INF/properties.xml shows how to configure the respective operators. + +### How to compile +`shell> mvn clean package` + +This will generate application package filter-1.0-SNAPSHOT.apa inside target directory. + +### How to run +Use the application package generated above to launch the application from UI console(if available) or apex command line interface. + +`apex> launch target/filter-1.0-SNAPSHOT.apa` + +You may also the run the application in local mode within your IDE by simply running the method ApplicationTest.testApplication(). http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/XmlJavadocCommentsExtractor.xsl ---------------------------------------------------------------------- diff --git a/examples/filter/XmlJavadocCommentsExtractor.xsl b/examples/filter/XmlJavadocCommentsExtractor.xsl new file mode 100644 index 0000000..08075a9 --- /dev/null +++ b/examples/filter/XmlJavadocCommentsExtractor.xsl @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + +<!-- + Document : XmlJavadocCommentsExtractor.xsl + Created on : September 16, 2014, 11:30 AM + Description: + The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="1.0"> + <xsl:output method="xml" standalone="yes"/> + + <!-- copy xml by selecting only the following nodes, attributes and text --> + <xsl:template match="node()|text()|@*"> + <xsl:copy> + <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/> + </xsl:copy> + </xsl:template> + + <!-- Strip off the following paths from the selected xml --> + <xsl:template match="//root/package/interface/interface + |//root/package/interface/method/@qualified + |//root/package/class/interface + |//root/package/class/class + |//root/package/class/method/@qualified + |//root/package/class/field/@qualified" /> + + <xsl:strip-space elements="*"/> +</xsl:stylesheet> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/pom.xml ---------------------------------------------------------------------- diff --git a/examples/filter/pom.xml b/examples/filter/pom.xml new file mode 100644 index 0000000..9407818 --- /dev/null +++ b/examples/filter/pom.xml @@ -0,0 +1,312 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.datatorrent.tutorial</groupId> + <version>1.0-SNAPSHOT</version> + <artifactId>filter</artifactId> + <packaging>jar</packaging> + + <!-- change these to the appropriate values --> + <name>Filter Operator</name> + <description>Apex application demonstrating filter operator</description> + + <properties> + <!-- change this if you desire to use a different version of Apex Core --> + <apex.version>3.5.0</apex.version> + <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath> + <malhar.version>3.6.0</malhar.version> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <version>2.9</version> + <configuration> + <downloadSources>true</downloadSources> + </configuration> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.3</version> + <configuration> + <encoding>UTF-8</encoding> + <source>1.7</source> + <target>1.7</target> + <debug>true</debug> + <optimize>false</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + </configuration> + </plugin> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/deps</outputDirectory> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <executions> + <execution> + <id>app-package-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${project.artifactId}-${project.version}-apexapp</finalName> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>src/assemble/appPackage.xml</descriptor> + </descriptors> + <archiverConfig> + <defaultDirectoryMode>0755</defaultDirectoryMode> + </archiverConfig> + <archive> + <manifestEntries> + <Class-Path>${apex.apppackage.classpath}</Class-Path> + <DT-Engine-Version>${apex.version}</DT-Engine-Version> + <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id> + <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name> + <DT-App-Package-Version>${project.version}</DT-App-Package-Version> + <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name> + <DT-App-Package-Description>${project.description}</DT-App-Package-Description> + </manifestEntries> + </archive> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <phase>package</phase> + <configuration> + <target> + <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar" + tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + <execution> + <!-- create resource directory for xml javadoc--> + <id>createJavadocDirectory</id> + <phase>generate-resources</phase> + <configuration> + <tasks> + <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/> + <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/> + </tasks> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9.1</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-${project.version}.apa</file> + <type>apa</type> + </artifact> + </artifacts> + <skipAttach>false</skipAttach> + </configuration> + </execution> + </executions> + </plugin> + + <!-- generate javdoc --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <!-- generate xml javadoc --> + <execution> + <id>xml-doclet</id> + <phase>generate-resources</phase> + <goals> + <goal>javadoc</goal> + </goals> + <configuration> + <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet> + <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam> + <useStandardDocletOptions>false</useStandardDocletOptions> + <docletArtifact> + <groupId>com.github.markusbernhardt</groupId> + <artifactId>xml-doclet</artifactId> + <version>1.0.4</version> + </docletArtifact> + </configuration> + </execution> + </executions> + </plugin> + <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags--> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>xml-maven-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>transform-xmljavadoc</id> + <phase>generate-resources</phase> + <goals> + <goal>transform</goal> + </goals> + </execution> + </executions> + <configuration> + <transformationSets> + <transformationSet> + <dir>${project.build.directory}/generated-resources/xml-javadoc</dir> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet> + <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir> + </transformationSet> + </transformationSets> + </configuration> + </plugin> + <!-- copy xml javadoc to class jar --> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>process-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/target/classes</outputDirectory> + <resources> + <resource> + <directory>${project.build.directory}/generated-resources/xml-javadoc</directory> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + + </build> + + <dependencies> + <!-- add your dependencies here --> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-library</artifactId> + <version>${malhar.version}</version> + <!-- + If you know that your application does not need transitive dependencies pulled in by malhar-library, + uncomment the following to reduce the size of your app package. + --> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.1</version> + <type>jar</type> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-contrib</artifactId> + <version>${malhar.version}</version> + <!-- + If you know that your application does not need transitive dependencies pulled in by malhar-library, + uncomment the following to reduce the size of your app package. + --> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>net.sf.supercsv</groupId> + <artifactId>super-csv</artifactId> + <version>2.4.0</version> + <optional>true</optional> + </dependency> + + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-common</artifactId> + <version>${apex.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.10</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-engine</artifactId> + <version>${apex.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>commons-compiler</artifactId> + <version>2.7.8</version> + <type>jar</type> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>janino</artifactId> + <version>2.7.8</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/assemble/appPackage.xml ---------------------------------------------------------------------- diff --git a/examples/filter/src/assemble/appPackage.xml b/examples/filter/src/assemble/appPackage.xml new file mode 100644 index 0000000..7ad071c --- /dev/null +++ b/examples/filter/src/assemble/appPackage.xml @@ -0,0 +1,43 @@ +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>appPackage</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/target/</directory> + <outputDirectory>/app</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/target/deps</directory> + <outputDirectory>/lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/site/conf</directory> + <outputDirectory>/conf</outputDirectory> + <includes> + <include>*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/META-INF</directory> + <outputDirectory>/META-INF</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/app</directory> + <outputDirectory>/app</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/resources</directory> + <outputDirectory>/resources</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java new file mode 100644 index 0000000..4ebb153 --- /dev/null +++ b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/Application.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2016 DataTorrent, Inc. + * All rights reserved. + */ + +package com.datatorrent.tutorial.filter; + +import org.apache.apex.malhar.lib.fs.FSRecordReaderModule; +import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.DAG; +import com.datatorrent.api.StreamingApplication; +import com.datatorrent.api.annotation.ApplicationAnnotation; +import com.datatorrent.contrib.formatter.CsvFormatter; +import com.datatorrent.contrib.parser.CsvParser; +import com.datatorrent.lib.filter.FilterOperator; + +/** + * Simple application illustrating filter operator + */ +@ApplicationAnnotation(name="FilterExample") +public class Application implements StreamingApplication +{ + + @Override + public void populateDAG(DAG dag, Configuration conf) + { + + FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class); + CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class); + FilterOperator filterOperator = dag.addOperator("filterOperator", new FilterOperator()); + + CsvFormatter selectedFormatter = dag.addOperator("selectedFormatter", new CsvFormatter()); + CsvFormatter rejectedFormatter = dag.addOperator("rejectedFormatter", new CsvFormatter()); + + StringFileOutputOperator selectedOutput = dag.addOperator("selectedOutput", new StringFileOutputOperator()); + StringFileOutputOperator rejectedOutput = dag.addOperator("rejectedOutput", new StringFileOutputOperator()); + + dag.addStream("record", recordReader.records, csvParser.in); + dag.addStream("pojo", csvParser.out, filterOperator.input); + + dag.addStream("pojoSelected", filterOperator.truePort, selectedFormatter.in); + dag.addStream("pojoRejected", filterOperator.falsePort, rejectedFormatter.in); + + dag.addStream("csvSelected", selectedFormatter.out, selectedOutput.input); + dag.addStream("csvRejected", rejectedFormatter.out, rejectedOutput.input); + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java new file mode 100644 index 0000000..c0d8817 --- /dev/null +++ b/examples/filter/src/main/java/com/datatorrent/tutorial/filter/TransactionPOJO.java @@ -0,0 +1,64 @@ +/** + * Copyright (c) 2016 DataTorrent, Inc. + * All rights reserved. + */ + +package com.datatorrent.tutorial.filter; + +public class TransactionPOJO +{ + + private long trasactionId; + private double amount; + private long accountNumber; + private String type; + + public long getTrasactionId() + { + return trasactionId; + } + + public void setTrasactionId(long trasactionId) + { + this.trasactionId = trasactionId; + } + + public double getAmount() + { + return amount; + } + + public void setAmount(double amount) + { + this.amount = amount; + } + + public long getAccountNumber() + { + return accountNumber; + } + + public void setAccountNumber(long accountNumber) + { + this.accountNumber = accountNumber; + } + + public String getType() + { + return type; + } + + public void setType(String type) + { + this.type = type; + } + + @Override + public String toString() + { + return "TransactionPOJO [trasactionId=" + trasactionId + ", amount=" + amount + ", accountNumber=" + accountNumber + + ", type=" + type + "]"; + } + + +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/input.txt ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/resources/META-INF/input.txt b/examples/filter/src/main/resources/META-INF/input.txt new file mode 100644 index 0000000..f87a772 --- /dev/null +++ b/examples/filter/src/main/resources/META-INF/input.txt @@ -0,0 +1,401 @@ +10001|84804533|51.0|DEBIT +10002|529683455|8637.0|CREDIT +10003|34646572|7443.0|CREDIT +10004|967805584|7342.0|CREDIT +10005|89940328|5100.0|DEBIT +10006|484971385|4559.0|DEBIT +10007|168472218|3539.0|CREDIT +10008|560263718|1624.0|CREDIT +10009|27097195|2471.0|DEBIT +10010|562944412|5143.0|DEBIT +10011|228473379|8336.0|DEBIT +10012|138798397|5595.0|CREDIT +10013|522515858|6054.0|CREDIT +10014|358415062|1886.0|CREDIT +10015|596239488|9936.0|DEBIT +10016|378386029|4623.0|DEBIT +10017|329310325|9064.0|CREDIT +10018|652183672|2748.0|DEBIT +10019|92850307|7752.0|DEBIT +10020|608582237|2329.0|CREDIT +10021|50283707|9987.0|DEBIT +10022|540706222|8639.0|CREDIT +10023|373122065|4834.0|DEBIT +10024|215520537|5684.0|DEBIT +10025|766304418|3561.0|CREDIT +10026|467948355|2802.0|CREDIT +10027|180522562|8662.0|DEBIT +10028|444946324|7783.0|CREDIT +10029|994074259|7125.0|CREDIT +10030|182740938|3113.0|CREDIT +10031|614671827|24979.0|CREDIT +10032|88726012|3603.0|CREDIT +10033|571353545|4157.0|CREDIT +10034|748258207|5329.0|CREDIT +10035|228328193|6883.0|DEBIT +10036|953766573|22516.0|DEBIT +10037|461283441|431.0|CREDIT +10038|518514310|1586.0|CREDIT +10039|381773726|8684.0|CREDIT +10040|680364582|4221.0|CREDIT +10041|728080172|23189.0|DEBIT +10042|468126452|4990.0|DEBIT +10043|459970527|6188.0|CREDIT +10044|100934948|3422.0|CREDIT +10045|31728881|435.0|DEBIT +10046|906868986|26145.0|CREDIT +10047|950687958|5002.0|CREDIT +10048|164869066|1487.0|CREDIT +10049|892759267|8102.0|DEBIT +10050|69946718|4464.0|CREDIT +10051|732833481|22186.0|DEBIT +10052|222384809|196.0|CREDIT +10053|619012622|5892.0|DEBIT +10054|524593769|8796.0|CREDIT +10055|360301464|9846.0|CREDIT +10056|76837426|21464.0|DEBIT +10057|894461469|3232.0|CREDIT +10058|905824283|9614.0|DEBIT +10059|201267525|6839.0|DEBIT +10060|650778374|3956.0|CREDIT +10061|539521241|26964.0|DEBIT +10062|257584130|2458.0|DEBIT +10063|185240027|4987.0|DEBIT +10064|340187828|6919.0|CREDIT +10065|167114400|758.0|DEBIT +10066|645317265|22942.0|CREDIT +10067|834054944|558.0|CREDIT +10068|4848158|2972.0|CREDIT +10069|611089049|3830.0|DEBIT +10070|574929455|7585.0|CREDIT +10071|23360143|22515.0|CREDIT +10072|678734877|1593.0|DEBIT +10073|465428415|4876.0|CREDIT +10074|631548543|4502.0|CREDIT +10075|708806494|954.0|DEBIT +10076|656870588|26892.0|DEBIT +10077|919361479|7612.0|DEBIT +10078|831562086|9544.0|DEBIT +10079|564566420|5850.0|DEBIT +10080|732363069|1502.0|CREDIT +10081|817401315|20419.0|CREDIT +10082|498367448|6829.0|CREDIT +10083|387288387|4453.0|DEBIT +10084|719081200|2961.0|CREDIT +10085|735565137|7400.0|DEBIT +10086|973661932|24358.0|CREDIT +10087|371001709|8504.0|CREDIT +10088|793778660|6867.0|DEBIT +10089|620050993|1582.0|CREDIT +10090|367541932|2858.0|DEBIT +10091|146470368|27897.0|DEBIT +10092|534083940|9915.0|CREDIT +10093|171218647|9262.0|DEBIT +10094|566590232|1904.0|DEBIT +10095|940994162|2697.0|CREDIT +10096|683965112|29656.0|CREDIT +10097|599783690|7551.0|CREDIT +10098|472038054|7107.0|CREDIT +10099|462438976|8978.0|CREDIT +10100|419462514|5799.0|CREDIT +10101|318680824|20128.0|CREDIT +10102|209434250|4813.0|DEBIT +10103|869224102|6504.0|DEBIT +10104|725686791|11.0|CREDIT +10105|781644695|1705.0|DEBIT +10106|48125174|22815.0|DEBIT +10107|857136119|2809.0|CREDIT +10108|290114536|7476.0|CREDIT +10109|90800192|2512.0|DEBIT +10110|884687225|1396.0|CREDIT +10111|780970052|24959.0|DEBIT +10112|976868472|8363.0|DEBIT +10113|570179873|7231.0|CREDIT +10114|399863103|4291.0|DEBIT +10115|358440857|5322.0|CREDIT +10116|713800593|21282.0|CREDIT +10117|50382939|6544.0|CREDIT +10118|42616440|5372.0|CREDIT +10119|220722302|3191.0|DEBIT +10120|676925763|9413.0|CREDIT +10121|66746026|21667.0|DEBIT +10122|412477439|9943.0|DEBIT +10123|405618613|2350.0|DEBIT +10124|415714190|9245.0|CREDIT +10125|563760753|7092.0|CREDIT +10126|737861484|23370.0|CREDIT +10127|434529942|7629.0|CREDIT +10128|514574310|3897.0|DEBIT +10129|27419748|8440.0|CREDIT +10130|165864941|9762.0|DEBIT +10131|571929475|26224.0|DEBIT +10132|52768757|3941.0|CREDIT +10133|734355265|621.0|CREDIT +10134|126968199|1286.0|DEBIT +10135|585441472|1770.0|DEBIT +10136|893117252|27637.0|CREDIT +10137|346242993|1487.0|CREDIT +10138|490611170|6440.0|DEBIT +10139|801216167|1515.0|DEBIT +10140|659265220|4552.0|CREDIT +10141|953467032|23295.0|CREDIT +10142|245703913|3093.0|DEBIT +10143|2247312|5209.0|DEBIT +10144|519787641|9527.0|DEBIT +10145|912166127|2172.0|DEBIT +10146|95502398|28322.0|DEBIT +10147|582275337|8712.0|DEBIT +10148|259281594|5252.0|CREDIT +10149|694423378|2060.0|DEBIT +10150|629554298|5967.0|CREDIT +10151|323058477|27765.0|DEBIT +10152|53101351|4382.0|CREDIT +10153|323164961|5328.0|CREDIT +10154|179064326|4341.0|DEBIT +10155|52433644|7812.0|DEBIT +10156|3744164|22345.0|DEBIT +10157|869913610|5813.0|CREDIT +10158|347708099|6201.0|CREDIT +10159|423486262|4871.0|DEBIT +10160|461540100|9256.0|DEBIT +10161|278026136|25472.0|CREDIT +10162|798468803|9224.0|DEBIT +10163|418877075|519.0|DEBIT +10164|40368770|245.0|DEBIT +10165|463671480|5775.0|CREDIT +10166|410182527|24299.0|DEBIT +10167|256176469|8602.0|DEBIT +10168|739490131|3012.0|CREDIT +10169|845152384|8188.0|DEBIT +10170|425841395|2578.0|DEBIT +10171|676685046|24621.0|CREDIT +10172|226331065|6325.0|DEBIT +10173|244147717|6282.0|DEBIT +10174|838618959|2970.0|CREDIT +10175|966466342|6090.0|DEBIT +10176|347315258|22401.0|CREDIT +10177|960290189|2392.0|DEBIT +10178|665504299|5721.0|DEBIT +10179|207166700|4961.0|DEBIT +10180|910728570|7626.0|CREDIT +10181|495476490|23835.0|CREDIT +10182|163142417|1630.0|CREDIT +10183|662313571|6927.0|DEBIT +10184|105583140|7757.0|DEBIT +10185|639328437|7461.0|DEBIT +10186|353744821|24419.0|CREDIT +10187|922940528|6379.0|CREDIT +10188|3320970|48.0|DEBIT +10189|217812412|7170.0|DEBIT +10190|660688167|8493.0|CREDIT +10191|319041088|27622.0|CREDIT +10192|29286525|4470.0|CREDIT +10193|170950644|5463.0|DEBIT +10194|813477150|376.0|CREDIT +10195|180786030|6536.0|DEBIT +10196|200090142|24312.0|CREDIT +10197|517613853|4244.0|CREDIT +10198|142109351|4045.0|CREDIT +10199|120408343|1224.0|CREDIT +10200|372437363|2587.0|CREDIT +10201|988342695|28896.0|DEBIT +10202|691849636|4285.0|CREDIT +10203|654204917|3074.0|DEBIT +10204|373303209|3566.0|CREDIT +10205|28536699|3681.0|CREDIT +10206|332098186|20324.0|DEBIT +10207|703547342|6351.0|DEBIT +10208|117707726|8618.0|CREDIT +10209|240155449|5987.0|DEBIT +10210|539240301|6073.0|CREDIT +10211|304868598|29587.0|DEBIT +10212|120948449|7938.0|CREDIT +10213|968947065|2045.0|DEBIT +10214|951562769|1738.0|CREDIT +10215|136201581|1112.0|DEBIT +10216|446120114|27378.0|CREDIT +10217|863802751|6610.0|DEBIT +10218|855970512|354.0|CREDIT +10219|339900628|4677.0|CREDIT +10220|706630053|3682.0|DEBIT +10221|904298621|27819.0|CREDIT +10222|174008949|3797.0|CREDIT +10223|546023627|7130.0|CREDIT +10224|138693364|4482.0|CREDIT +10225|698667700|1055.0|CREDIT +10226|286039255|23856.0|CREDIT +10227|597215884|6851.0|DEBIT +10228|841851199|3901.0|CREDIT +10229|658855748|4700.0|DEBIT +10230|741198758|8011.0|CREDIT +10231|659386308|29955.0|DEBIT +10232|130516026|8200.0|CREDIT +10233|255513743|877.0|CREDIT +10234|235986880|5608.0|DEBIT +10235|755745324|4888.0|CREDIT +10236|864054476|25873.0|DEBIT +10237|117830905|4932.0|CREDIT +10238|436598482|3043.0|DEBIT +10239|415570756|5563.0|DEBIT +10240|160904677|5698.0|DEBIT +10241|12110227|25870.0|CREDIT +10242|325540634|3633.0|CREDIT +10243|934407707|9198.0|DEBIT +10244|970044154|253.0|CREDIT +10245|276211772|2484.0|DEBIT +10246|632081618|22307.0|DEBIT +10247|515392682|3085.0|DEBIT +10248|106926205|9599.0|CREDIT +10249|403051823|6598.0|DEBIT +10250|739699283|2898.0|CREDIT +10251|913373304|25760.0|CREDIT +10252|588855751|4159.0|CREDIT +10253|433188311|8895.0|CREDIT +10254|613207771|2386.0|CREDIT +10255|529952498|2469.0|CREDIT +10256|487018926|27501.0|DEBIT +10257|788255680|4566.0|CREDIT +10258|177961056|4417.0|DEBIT +10259|856270849|7959.0|DEBIT +10260|362871038|2661.0|DEBIT +10261|552260828|20328.0|DEBIT +10262|51438711|7238.0|CREDIT +10263|550177844|6044.0|DEBIT +10264|796710293|3723.0|CREDIT +10265|747419280|353.0|CREDIT +10266|625233460|29396.0|DEBIT +10267|502287841|7040.0|DEBIT +10268|868155169|8175.0|DEBIT +10269|217771255|6130.0|DEBIT +10270|983601385|9798.0|CREDIT +10271|676948266|29532.0|DEBIT +10272|931578961|6792.0|DEBIT +10273|627410737|6313.0|CREDIT +10274|227280958|6610.0|DEBIT +10275|814312724|2083.0|DEBIT +10276|614444626|24522.0|CREDIT +10277|734388101|3870.0|CREDIT +10278|969611064|344.0|DEBIT +10279|955987400|2897.0|DEBIT +10280|742505356|8610.0|CREDIT +10281|522681680|22593.0|CREDIT +10282|381786126|4002.0|CREDIT +10283|466001485|8284.0|CREDIT +10284|310654102|4678.0|DEBIT +10285|437156002|7311.0|CREDIT +10286|495903944|27075.0|DEBIT +10287|503495897|6757.0|CREDIT +10288|411513405|263.0|CREDIT +10289|143598653|1820.0|DEBIT +10290|702368818|4650.0|CREDIT +10291|248658152|28021.0|DEBIT +10292|545322914|7008.0|DEBIT +10293|176940492|584.0|CREDIT +10294|592922722|5374.0|DEBIT +10295|255199017|2002.0|DEBIT +10296|858007111|24374.0|CREDIT +10297|103227713|6796.0|DEBIT +10298|366884380|2496.0|DEBIT +10299|147239319|9209.0|CREDIT +10300|137542451|6119.0|DEBIT +10301|794527271|26028.0|DEBIT +10302|399318593|4910.0|DEBIT +10303|568931105|7331.0|CREDIT +10304|144716426|1469.0|CREDIT +10305|603015441|4052.0|CREDIT +10306|293223789|20876.0|CREDIT +10307|606444995|7826.0|DEBIT +10308|400150434|746.0|DEBIT +10309|767683056|3244.0|DEBIT +10310|876098616|6538.0|DEBIT +10311|74081563|25362.0|DEBIT +10312|711049958|2091.0|DEBIT +10313|657718602|5381.0|CREDIT +10314|912202073|6467.0|DEBIT +10315|654099636|2938.0|DEBIT +10316|241063831|21767.0|DEBIT +10317|858815265|6203.0|CREDIT +10318|939864847|9961.0|CREDIT +10319|949717040|5242.0|DEBIT +10320|204238472|9615.0|CREDIT +10321|866409326|22291.0|DEBIT +10322|949996114|7868.0|DEBIT +10323|422911083|5645.0|CREDIT +10324|322526780|3789.0|DEBIT +10325|133951099|5756.0|DEBIT +10326|591884483|29458.0|CREDIT +10327|695253171|3670.0|CREDIT +10328|215281673|5174.0|DEBIT +10329|436311786|5423.0|CREDIT +10330|518432518|9037.0|CREDIT +10331|425349532|26548.0|DEBIT +10332|837790968|8858.0|CREDIT +10333|887625960|6337.0|CREDIT +10334|757883576|6954.0|CREDIT +10335|459944677|3951.0|DEBIT +10336|801117092|22688.0|CREDIT +10337|516885021|9358.0|DEBIT +10338|232560679|8366.0|CREDIT +10339|611885533|4186.0|DEBIT +10340|689162123|7565.0|CREDIT +10341|501197134|25917.0|DEBIT +10342|334174445|617.0|CREDIT +10343|180187633|4945.0|CREDIT +10344|887467274|5195.0|DEBIT +10345|808750192|1136.0|DEBIT +10346|13639891|21796.0|CREDIT +10347|638906135|8026.0|DEBIT +10348|834147190|1571.0|DEBIT +10349|22124303|4402.0|CREDIT +10350|235076790|2772.0|CREDIT +10351|552645711|21462.0|DEBIT +10352|389278238|9606.0|CREDIT +10353|895551786|3928.0|DEBIT +10354|584140777|8246.0|CREDIT +10355|593368904|6750.0|CREDIT +10356|329314918|27229.0|DEBIT +10357|483588760|5955.0|DEBIT +10358|827410725|2563.0|DEBIT +10359|932975398|3319.0|DEBIT +10360|976026166|5742.0|DEBIT +10361|823102894|29658.0|CREDIT +10362|711407791|2859.0|DEBIT +10363|815218938|5248.0|DEBIT +10364|406037352|9096.0|DEBIT +10365|653047392|1314.0|CREDIT +10366|625297640|25451.0|CREDIT +10367|726603183|97.0|CREDIT +10368|530543948|2115.0|CREDIT +10369|935474118|2615.0|CREDIT +10370|455235000|1673.0|CREDIT +10371|24584042|24892.0|CREDIT +10372|827942510|9186.0|CREDIT +10373|999662321|3634.0|DEBIT +10374|634102829|324.0|CREDIT +10375|57740902|1549.0|DEBIT +10376|908335261|29778.0|CREDIT +10377|208503014|5634.0|DEBIT +10378|886641515|9910.0|DEBIT +10379|899684657|5835.0|DEBIT +10380|953557084|3530.0|CREDIT +10381|6573510|28709.0|DEBIT +10382|7477110|835.0|DEBIT +10383|347810136|3494.0|DEBIT +10384|434772334|4891.0|DEBIT +10385|33987507|4190.0|DEBIT +10386|347989457|28234.0|CREDIT +10387|595056330|5993.0|CREDIT +10388|770726182|8415.0|CREDIT +10389|358147886|7501.0|CREDIT +10390|722150002|6400.0|DEBIT +10391|275383311|27191.0|DEBIT +10392|230539494|8684.0|CREDIT +10393|326949401|2621.0|CREDIT +10394|514140841|5321.0|DEBIT +10395|117582460|4431.0|DEBIT +10396|401338491|24864.0|CREDIT +10397|661347779|8126.0|CREDIT +10398|512723286|5761.0|DEBIT +10399|435455695|8425.0|DEBIT +10400|379646166|7155.0|DEBIT + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/properties.xml ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/resources/META-INF/properties.xml b/examples/filter/src/main/resources/META-INF/properties.xml new file mode 100644 index 0000000..079fb3f --- /dev/null +++ b/examples/filter/src/main/resources/META-INF/properties.xml @@ -0,0 +1,95 @@ +<?xml version="1.0"?> +<configuration> + <!-- + <property> + <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name> + <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value> + </property> + --> + <!-- memory assigned to app master + <property> + <name>dt.attr.MASTER_MEMORY_MB</name> + <value>1024</value> + </property> + --> + + <!-- these values override those set in populateDAG --> + + <property> + <name>dt.application.FilterExample.operator.*.attr.MEMORY_MB</name> + <value>750</value> + </property> + <property> + <name>dt.application.FilterExample.attr.CHECKPOINT_WINDOW_COUNT</name> + <value>10</value> + </property> + <property> + <name>dt.application.FilterExample.operator.recordReader.prop.files</name> + <value>src/main/resources/META-INF/input.txt</value> + </property> + <property> + <name>dt.application.FilterExample.operator.csvParser.prop.schema</name> + <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value> + </property> + <property> + <name>dt.application.FilterExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name> + <value>com.datatorrent.tutorial.filter.TransactionPOJO</value> + </property> + <property> + <name>dt.application.FilterExample.operator.selectedOutput.prop.maxIdleWindows</name> + <value>20</value> + </property> + <property> + <name>dt.application.FilterExample.operator.rejectedOutput.prop.maxIdleWindows</name> + <value>20</value> + </property> + <property> + <name>dt.application.FilterExample.operator.filterOperator.port.input.attr.TUPLE_CLASS</name> + <value>com.datatorrent.tutorial.filter.TransactionPOJO</value> + </property> + <property> + <name>dt.application.FilterExample.operator.selectedFormatter.port.in.attr.TUPLE_CLASS</name> + <value>com.datatorrent.tutorial.filter.TransactionPOJO</value> + </property> + <property> + <name>dt.application.FilterExample.operator.rejectedFormatter.port.in.attr.TUPLE_CLASS</name> + <value>com.datatorrent.tutorial.filter.TransactionPOJO</value> + </property> + <property> + <name>dt.application.FilterExample.operator.filterOperator.prop.condition</name> + <value>({$}.getAmount() >= 20000)</value> + </property> + <property> + <name>dt.application.FilterExample.operator.filterOperator.prop.expressionFunctionsItem[5]</name> + <value>org.apache.commons.lang3.BooleanUtils.*</value> + </property> + <property> + <name>dt.application.FilterExample.operator.selectedFormatter.prop.schema</name> + <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value> + </property> + <property> + <name>dt.application.FilterExample.operator.rejectedFormatter.prop.schema</name> + <value>{"separator": "|","quoteChar":"\"","lineDelimiter":"","fields": [{"name": "trasactionId","type": "long"},{"name": "accountNumber","type": "long"},{"name": "amount","type": "double"},{"name": "type","type": "String"}]}</value> + </property> + <property> + <name>dt.application.FilterExample.operator.selectedOutput.prop.filePath</name> + <value>file:///tmp</value> + </property> + <property> + <name>dt.application.FilterExample.operator.selectedOutput.prop.outputFileName</name> + <value>selected.txt</value> + </property> + <property> + <name>dt.application.FilterExample.operator.rejectedOutput.prop.filePath</name> + <value>file:///tmp</value> + </property> + <property> + <name>dt.application.FilterExample.operator.rejectedOutput.prop.outputFileName</name> + <value>rejected.txt</value> + </property> + <property> + <name>dt.loggers.level</name> + <value>com.datatorrent.*:DEBUG,org.apache.apex.malhar.*:DEBUG</value> + </property> +</configuration> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/rejected_output.txt ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/resources/META-INF/rejected_output.txt b/examples/filter/src/main/resources/META-INF/rejected_output.txt new file mode 100644 index 0000000..d83b7a9 --- /dev/null +++ b/examples/filter/src/main/resources/META-INF/rejected_output.txt @@ -0,0 +1,326 @@ +10001|84804533|51.0|DEBIT +10002|529683455|8637.0|CREDIT +10003|34646572|7443.0|CREDIT +10004|967805584|7342.0|CREDIT +10005|89940328|5100.0|DEBIT +10006|484971385|4559.0|DEBIT +10007|168472218|3539.0|CREDIT +10008|560263718|1624.0|CREDIT +10009|27097195|2471.0|DEBIT +10010|562944412|5143.0|DEBIT +10011|228473379|8336.0|DEBIT +10012|138798397|5595.0|CREDIT +10013|522515858|6054.0|CREDIT +10014|358415062|1886.0|CREDIT +10015|596239488|9936.0|DEBIT +10016|378386029|4623.0|DEBIT +10017|329310325|9064.0|CREDIT +10018|652183672|2748.0|DEBIT +10019|92850307|7752.0|DEBIT +10020|608582237|2329.0|CREDIT +10021|50283707|9987.0|DEBIT +10022|540706222|8639.0|CREDIT +10023|373122065|4834.0|DEBIT +10024|215520537|5684.0|DEBIT +10025|766304418|3561.0|CREDIT +10026|467948355|2802.0|CREDIT +10027|180522562|8662.0|DEBIT +10028|444946324|7783.0|CREDIT +10029|994074259|7125.0|CREDIT +10030|182740938|3113.0|CREDIT +10032|88726012|3603.0|CREDIT +10033|571353545|4157.0|CREDIT +10034|748258207|5329.0|CREDIT +10035|228328193|6883.0|DEBIT +10037|461283441|431.0|CREDIT +10038|518514310|1586.0|CREDIT +10039|381773726|8684.0|CREDIT +10040|680364582|4221.0|CREDIT +10042|468126452|4990.0|DEBIT +10043|459970527|6188.0|CREDIT +10044|100934948|3422.0|CREDIT +10045|31728881|435.0|DEBIT +10047|950687958|5002.0|CREDIT +10048|164869066|1487.0|CREDIT +10049|892759267|8102.0|DEBIT +10050|69946718|4464.0|CREDIT +10052|222384809|196.0|CREDIT +10053|619012622|5892.0|DEBIT +10054|524593769|8796.0|CREDIT +10055|360301464|9846.0|CREDIT +10057|894461469|3232.0|CREDIT +10058|905824283|9614.0|DEBIT +10059|201267525|6839.0|DEBIT +10060|650778374|3956.0|CREDIT +10062|257584130|2458.0|DEBIT +10063|185240027|4987.0|DEBIT +10064|340187828|6919.0|CREDIT +10065|167114400|758.0|DEBIT +10067|834054944|558.0|CREDIT +10068|4848158|2972.0|CREDIT +10069|611089049|3830.0|DEBIT +10070|574929455|7585.0|CREDIT +10072|678734877|1593.0|DEBIT +10073|465428415|4876.0|CREDIT +10074|631548543|4502.0|CREDIT +10075|708806494|954.0|DEBIT +10077|919361479|7612.0|DEBIT +10078|831562086|9544.0|DEBIT +10079|564566420|5850.0|DEBIT +10080|732363069|1502.0|CREDIT +10082|498367448|6829.0|CREDIT +10083|387288387|4453.0|DEBIT +10084|719081200|2961.0|CREDIT +10085|735565137|7400.0|DEBIT +10087|371001709|8504.0|CREDIT +10088|793778660|6867.0|DEBIT +10089|620050993|1582.0|CREDIT +10090|367541932|2858.0|DEBIT +10092|534083940|9915.0|CREDIT +10093|171218647|9262.0|DEBIT +10094|566590232|1904.0|DEBIT +10095|940994162|2697.0|CREDIT +10097|599783690|7551.0|CREDIT +10098|472038054|7107.0|CREDIT +10099|462438976|8978.0|CREDIT +10100|419462514|5799.0|CREDIT +10102|209434250|4813.0|DEBIT +10103|869224102|6504.0|DEBIT +10104|725686791|11.0|CREDIT +10105|781644695|1705.0|DEBIT +10107|857136119|2809.0|CREDIT +10108|290114536|7476.0|CREDIT +10109|90800192|2512.0|DEBIT +10110|884687225|1396.0|CREDIT +10112|976868472|8363.0|DEBIT +10113|570179873|7231.0|CREDIT +10114|399863103|4291.0|DEBIT +10115|358440857|5322.0|CREDIT +10117|50382939|6544.0|CREDIT +10118|42616440|5372.0|CREDIT +10119|220722302|3191.0|DEBIT +10120|676925763|9413.0|CREDIT +10122|412477439|9943.0|DEBIT +10123|405618613|2350.0|DEBIT +10124|415714190|9245.0|CREDIT +10125|563760753|7092.0|CREDIT +10127|434529942|7629.0|CREDIT +10128|514574310|3897.0|DEBIT +10129|27419748|8440.0|CREDIT +10130|165864941|9762.0|DEBIT +10132|52768757|3941.0|CREDIT +10133|734355265|621.0|CREDIT +10134|126968199|1286.0|DEBIT +10135|585441472|1770.0|DEBIT +10137|346242993|1487.0|CREDIT +10138|490611170|6440.0|DEBIT +10139|801216167|1515.0|DEBIT +10140|659265220|4552.0|CREDIT +10142|245703913|3093.0|DEBIT +10143|2247312|5209.0|DEBIT +10144|519787641|9527.0|DEBIT +10145|912166127|2172.0|DEBIT +10147|582275337|8712.0|DEBIT +10148|259281594|5252.0|CREDIT +10149|694423378|2060.0|DEBIT +10150|629554298|5967.0|CREDIT +10152|53101351|4382.0|CREDIT +10153|323164961|5328.0|CREDIT +10154|179064326|4341.0|DEBIT +10155|52433644|7812.0|DEBIT +10157|869913610|5813.0|CREDIT +10158|347708099|6201.0|CREDIT +10159|423486262|4871.0|DEBIT +10160|461540100|9256.0|DEBIT +10162|798468803|9224.0|DEBIT +10163|418877075|519.0|DEBIT +10164|40368770|245.0|DEBIT +10165|463671480|5775.0|CREDIT +10167|256176469|8602.0|DEBIT +10168|739490131|3012.0|CREDIT +10169|845152384|8188.0|DEBIT +10170|425841395|2578.0|DEBIT +10172|226331065|6325.0|DEBIT +10173|244147717|6282.0|DEBIT +10174|838618959|2970.0|CREDIT +10175|966466342|6090.0|DEBIT +10177|960290189|2392.0|DEBIT +10178|665504299|5721.0|DEBIT +10179|207166700|4961.0|DEBIT +10180|910728570|7626.0|CREDIT +10182|163142417|1630.0|CREDIT +10183|662313571|6927.0|DEBIT +10184|105583140|7757.0|DEBIT +10185|639328437|7461.0|DEBIT +10187|922940528|6379.0|CREDIT +10188|3320970|48.0|DEBIT +10189|217812412|7170.0|DEBIT +10190|660688167|8493.0|CREDIT +10192|29286525|4470.0|CREDIT +10193|170950644|5463.0|DEBIT +10194|813477150|376.0|CREDIT +10195|180786030|6536.0|DEBIT +10197|517613853|4244.0|CREDIT +10198|142109351|4045.0|CREDIT +10199|120408343|1224.0|CREDIT +10200|372437363|2587.0|CREDIT +10202|691849636|4285.0|CREDIT +10203|654204917|3074.0|DEBIT +10204|373303209|3566.0|CREDIT +10205|28536699|3681.0|CREDIT +10207|703547342|6351.0|DEBIT +10208|117707726|8618.0|CREDIT +10209|240155449|5987.0|DEBIT +10210|539240301|6073.0|CREDIT +10212|120948449|7938.0|CREDIT +10213|968947065|2045.0|DEBIT +10214|951562769|1738.0|CREDIT +10215|136201581|1112.0|DEBIT +10217|863802751|6610.0|DEBIT +10218|855970512|354.0|CREDIT +10219|339900628|4677.0|CREDIT +10220|706630053|3682.0|DEBIT +10222|174008949|3797.0|CREDIT +10223|546023627|7130.0|CREDIT +10224|138693364|4482.0|CREDIT +10225|698667700|1055.0|CREDIT +10227|597215884|6851.0|DEBIT +10228|841851199|3901.0|CREDIT +10229|658855748|4700.0|DEBIT +10230|741198758|8011.0|CREDIT +10232|130516026|8200.0|CREDIT +10233|255513743|877.0|CREDIT +10234|235986880|5608.0|DEBIT +10235|755745324|4888.0|CREDIT +10237|117830905|4932.0|CREDIT +10238|436598482|3043.0|DEBIT +10239|415570756|5563.0|DEBIT +10240|160904677|5698.0|DEBIT +10242|325540634|3633.0|CREDIT +10243|934407707|9198.0|DEBIT +10244|970044154|253.0|CREDIT +10245|276211772|2484.0|DEBIT +10247|515392682|3085.0|DEBIT +10248|106926205|9599.0|CREDIT +10249|403051823|6598.0|DEBIT +10250|739699283|2898.0|CREDIT +10252|588855751|4159.0|CREDIT +10253|433188311|8895.0|CREDIT +10254|613207771|2386.0|CREDIT +10255|529952498|2469.0|CREDIT +10257|788255680|4566.0|CREDIT +10258|177961056|4417.0|DEBIT +10259|856270849|7959.0|DEBIT +10260|362871038|2661.0|DEBIT +10262|51438711|7238.0|CREDIT +10263|550177844|6044.0|DEBIT +10264|796710293|3723.0|CREDIT +10265|747419280|353.0|CREDIT +10267|502287841|7040.0|DEBIT +10268|868155169|8175.0|DEBIT +10269|217771255|6130.0|DEBIT +10270|983601385|9798.0|CREDIT +10272|931578961|6792.0|DEBIT +10273|627410737|6313.0|CREDIT +10274|227280958|6610.0|DEBIT +10275|814312724|2083.0|DEBIT +10277|734388101|3870.0|CREDIT +10278|969611064|344.0|DEBIT +10279|955987400|2897.0|DEBIT +10280|742505356|8610.0|CREDIT +10282|381786126|4002.0|CREDIT +10283|466001485|8284.0|CREDIT +10284|310654102|4678.0|DEBIT +10285|437156002|7311.0|CREDIT +10287|503495897|6757.0|CREDIT +10288|411513405|263.0|CREDIT +10289|143598653|1820.0|DEBIT +10290|702368818|4650.0|CREDIT +10292|545322914|7008.0|DEBIT +10293|176940492|584.0|CREDIT +10294|592922722|5374.0|DEBIT +10295|255199017|2002.0|DEBIT +10297|103227713|6796.0|DEBIT +10298|366884380|2496.0|DEBIT +10299|147239319|9209.0|CREDIT +10300|137542451|6119.0|DEBIT +10302|399318593|4910.0|DEBIT +10303|568931105|7331.0|CREDIT +10304|144716426|1469.0|CREDIT +10305|603015441|4052.0|CREDIT +10307|606444995|7826.0|DEBIT +10308|400150434|746.0|DEBIT +10309|767683056|3244.0|DEBIT +10310|876098616|6538.0|DEBIT +10312|711049958|2091.0|DEBIT +10313|657718602|5381.0|CREDIT +10314|912202073|6467.0|DEBIT +10315|654099636|2938.0|DEBIT +10317|858815265|6203.0|CREDIT +10318|939864847|9961.0|CREDIT +10319|949717040|5242.0|DEBIT +10320|204238472|9615.0|CREDIT +10322|949996114|7868.0|DEBIT +10323|422911083|5645.0|CREDIT +10324|322526780|3789.0|DEBIT +10325|133951099|5756.0|DEBIT +10327|695253171|3670.0|CREDIT +10328|215281673|5174.0|DEBIT +10329|436311786|5423.0|CREDIT +10330|518432518|9037.0|CREDIT +10332|837790968|8858.0|CREDIT +10333|887625960|6337.0|CREDIT +10334|757883576|6954.0|CREDIT +10335|459944677|3951.0|DEBIT +10337|516885021|9358.0|DEBIT +10338|232560679|8366.0|CREDIT +10339|611885533|4186.0|DEBIT +10340|689162123|7565.0|CREDIT +10342|334174445|617.0|CREDIT +10343|180187633|4945.0|CREDIT +10344|887467274|5195.0|DEBIT +10345|808750192|1136.0|DEBIT +10347|638906135|8026.0|DEBIT +10348|834147190|1571.0|DEBIT +10349|22124303|4402.0|CREDIT +10350|235076790|2772.0|CREDIT +10352|389278238|9606.0|CREDIT +10353|895551786|3928.0|DEBIT +10354|584140777|8246.0|CREDIT +10355|593368904|6750.0|CREDIT +10357|483588760|5955.0|DEBIT +10358|827410725|2563.0|DEBIT +10359|932975398|3319.0|DEBIT +10360|976026166|5742.0|DEBIT +10362|711407791|2859.0|DEBIT +10363|815218938|5248.0|DEBIT +10364|406037352|9096.0|DEBIT +10365|653047392|1314.0|CREDIT +10367|726603183|97.0|CREDIT +10368|530543948|2115.0|CREDIT +10369|935474118|2615.0|CREDIT +10370|455235000|1673.0|CREDIT +10372|827942510|9186.0|CREDIT +10373|999662321|3634.0|DEBIT +10374|634102829|324.0|CREDIT +10375|57740902|1549.0|DEBIT +10377|208503014|5634.0|DEBIT +10378|886641515|9910.0|DEBIT +10379|899684657|5835.0|DEBIT +10380|953557084|3530.0|CREDIT +10382|7477110|835.0|DEBIT +10383|347810136|3494.0|DEBIT +10384|434772334|4891.0|DEBIT +10385|33987507|4190.0|DEBIT +10387|595056330|5993.0|CREDIT +10388|770726182|8415.0|CREDIT +10389|358147886|7501.0|CREDIT +10390|722150002|6400.0|DEBIT +10392|230539494|8684.0|CREDIT +10393|326949401|2621.0|CREDIT +10394|514140841|5321.0|DEBIT +10395|117582460|4431.0|DEBIT +10397|661347779|8126.0|CREDIT +10398|512723286|5761.0|DEBIT +10399|435455695|8425.0|DEBIT +10400|379646166|7155.0|DEBIT http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/main/resources/META-INF/selected_output.txt ---------------------------------------------------------------------- diff --git a/examples/filter/src/main/resources/META-INF/selected_output.txt b/examples/filter/src/main/resources/META-INF/selected_output.txt new file mode 100644 index 0000000..887d721 --- /dev/null +++ b/examples/filter/src/main/resources/META-INF/selected_output.txt @@ -0,0 +1,74 @@ +10031|614671827|24979.0|CREDIT +10036|953766573|22516.0|DEBIT +10041|728080172|23189.0|DEBIT +10046|906868986|26145.0|CREDIT +10051|732833481|22186.0|DEBIT +10056|76837426|21464.0|DEBIT +10061|539521241|26964.0|DEBIT +10066|645317265|22942.0|CREDIT +10071|23360143|22515.0|CREDIT +10076|656870588|26892.0|DEBIT +10081|817401315|20419.0|CREDIT +10086|973661932|24358.0|CREDIT +10091|146470368|27897.0|DEBIT +10096|683965112|29656.0|CREDIT +10101|318680824|20128.0|CREDIT +10106|48125174|22815.0|DEBIT +10111|780970052|24959.0|DEBIT +10116|713800593|21282.0|CREDIT +10121|66746026|21667.0|DEBIT +10126|737861484|23370.0|CREDIT +10131|571929475|26224.0|DEBIT +10136|893117252|27637.0|CREDIT +10141|953467032|23295.0|CREDIT +10146|95502398|28322.0|DEBIT +10151|323058477|27765.0|DEBIT +10156|3744164|22345.0|DEBIT +10161|278026136|25472.0|CREDIT +10166|410182527|24299.0|DEBIT +10171|676685046|24621.0|CREDIT +10176|347315258|22401.0|CREDIT +10181|495476490|23835.0|CREDIT +10186|353744821|24419.0|CREDIT +10191|319041088|27622.0|CREDIT +10196|200090142|24312.0|CREDIT +10201|988342695|28896.0|DEBIT +10206|332098186|20324.0|DEBIT +10211|304868598|29587.0|DEBIT +10216|446120114|27378.0|CREDIT +10221|904298621|27819.0|CREDIT +10226|286039255|23856.0|CREDIT +10231|659386308|29955.0|DEBIT +10236|864054476|25873.0|DEBIT +10241|12110227|25870.0|CREDIT +10246|632081618|22307.0|DEBIT +10251|913373304|25760.0|CREDIT +10256|487018926|27501.0|DEBIT +10261|552260828|20328.0|DEBIT +10266|625233460|29396.0|DEBIT +10271|676948266|29532.0|DEBIT +10276|614444626|24522.0|CREDIT +10281|522681680|22593.0|CREDIT +10286|495903944|27075.0|DEBIT +10291|248658152|28021.0|DEBIT +10296|858007111|24374.0|CREDIT +10301|794527271|26028.0|DEBIT +10306|293223789|20876.0|CREDIT +10311|74081563|25362.0|DEBIT +10316|241063831|21767.0|DEBIT +10321|866409326|22291.0|DEBIT +10326|591884483|29458.0|CREDIT +10331|425349532|26548.0|DEBIT +10336|801117092|22688.0|CREDIT +10341|501197134|25917.0|DEBIT +10346|13639891|21796.0|CREDIT +10351|552645711|21462.0|DEBIT +10356|329314918|27229.0|DEBIT +10361|823102894|29658.0|CREDIT +10366|625297640|25451.0|CREDIT +10371|24584042|24892.0|CREDIT +10376|908335261|29778.0|CREDIT +10381|6573510|28709.0|DEBIT +10386|347989457|28234.0|CREDIT +10391|275383311|27191.0|DEBIT +10396|401338491|24864.0|CREDIT http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java ---------------------------------------------------------------------- diff --git a/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java b/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java new file mode 100644 index 0000000..a90e822 --- /dev/null +++ b/examples/filter/src/test/java/com/datatorrent/tutorial/filter/ApplicationTest.java @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2016 DataTorrent, Inc. + * All rights reserved. + */ + +package com.datatorrent.tutorial.filter; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.Callable; + +import javax.validation.ConstraintViolationException; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.LocalMode; +import com.datatorrent.stram.StramLocalCluster; + +/** + * Test the DAG declaration in local mode. + */ +public class ApplicationTest +{ + private String outputDir; + + public static class TestMeta extends TestWatcher + { + public String baseDirectory; + + @Override + protected void starting(org.junit.runner.Description description) + { + this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName(); + } + + @Override + protected void finished(Description description) + { + super.finished(description); + try { + FileUtils.forceDelete(new File(baseDirectory)); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + @Rule + public TestMeta testMeta = new TestMeta(); + + @Before + public void setup() throws Exception + { + outputDir = testMeta.baseDirectory + File.separator + "output"; + } + + @Test + public void testApplication() throws IOException, Exception + { + + try { + LocalMode lma = LocalMode.newInstance(); + Configuration conf = new Configuration(false); + conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml")); + conf.set("dt.application.FilterExample.operator.selectedOutput.prop.filePath", outputDir); + conf.set("dt.application.FilterExample.operator.rejectedOutput.prop.filePath", outputDir); + final File selectedfile = FileUtils.getFile(outputDir, "selected.txt_8.0"); + final File rejectedfile = FileUtils.getFile(outputDir, "rejected.txt_6.0"); + + lma.prepareDAG(new Application(), conf); + LocalMode.Controller lc = lma.getController(); + + ((StramLocalCluster)lc).setExitCondition(new Callable<Boolean>() + { + @Override + public Boolean call() throws Exception + { + if (selectedfile.exists() && rejectedfile.exists()) { + return true; + } + return false; + } + }); + + lc.run(40000); + Assert.assertTrue( + FileUtils.contentEquals( + FileUtils.getFile( + "src/main/resources/META-INF/selected_output.txt" + ),selectedfile)); + + Assert.assertTrue( + FileUtils.contentEquals( + FileUtils.getFile( + "src/main/resources/META-INF/rejected_output.txt" + ),rejectedfile)); + + } catch (ConstraintViolationException e) { + Assert.fail("constraint violations: " + e.getConstraintViolations()); + } + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/0590c258/examples/filter/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/examples/filter/src/test/resources/log4j.properties b/examples/filter/src/test/resources/log4j.properties new file mode 100644 index 0000000..98544e8 --- /dev/null +++ b/examples/filter/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +log4j.rootLogger=DEBUG,CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.RFA.File=/tmp/app.log + +# to enable, add SYSLOG to rootLogger +log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender +log4j.appender.SYSLOG.syslogHost=127.0.0.1 +log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout +log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n +log4j.appender.SYSLOG.Facility=LOCAL1 + +#log4j.logger.org.apache.commons.beanutils=warn +log4j.logger.com.datatorrent=debug +log4j.logger.org.apache.apex=debug +log4j.logger.org=info
