SPOI-8561 Record reader example app 2. Added entry for recordReader in README
3. Verification for number of records reaching to the output. 4. Using runAsync instead of run(10s) 5. Updated README 6. Incorporating review comments Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/11f37671 Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/11f37671 Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/11f37671 Branch: refs/heads/master Commit: 11f376711e35c456f2c8d2d846b1b6271495b82f Parents: 0590c25 Author: yogidevendra <[email protected]> Authored: Mon Jul 4 17:17:05 2016 +0530 Committer: Lakshmi Prasanna Velineni <[email protected]> Committed: Sun Mar 26 11:43:48 2017 -0700 ---------------------------------------------------------------------- examples/recordReader/README.md | 8 + .../XmlJavadocCommentsExtractor.xsl | 44 +++ examples/recordReader/pom.xml | 297 +++++++++++++++++++ .../recordReader/src/assemble/appPackage.xml | 43 +++ .../com/example/recordReader/Application.java | 32 ++ .../recordReader/TransactionsSchema.java | 168 +++++++++++ .../src/main/resources/META-INF/CustomerTxnData | 100 +++++++ .../src/main/resources/META-INF/properties.xml | 159 ++++++++++ .../example/recordReader/ApplicationTest.java | 91 ++++++ .../src/test/resources/log4j.properties | 21 ++ 10 files changed, 963 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/README.md ---------------------------------------------------------------------- diff --git a/examples/recordReader/README.md b/examples/recordReader/README.md new file mode 100644 index 0000000..ad715fc --- /dev/null +++ b/examples/recordReader/README.md @@ -0,0 +1,8 @@ +This example shows how to use `FSRecordReaderModule` to read newline delimited records from a file convert them to plain old java objects (POJO) using `CSVParser`. These POJOs are converted to String using `CsvFormatter` and then written to output file using `StringFileOutputOperator`. + +The properties file `META-INF/properties.xml` shows how to configure the respective operators. + +The application can be run on an actual cluster or in local mode within your IDE by +simply running the method `ApplicationTest.testApplication()`. + +One may tweak this example to add operator of their choice in between `CSVParser` and `CsvFormatter` to achieve functionality to suit your need. http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/XmlJavadocCommentsExtractor.xsl ---------------------------------------------------------------------- diff --git a/examples/recordReader/XmlJavadocCommentsExtractor.xsl b/examples/recordReader/XmlJavadocCommentsExtractor.xsl new file mode 100644 index 0000000..08075a9 --- /dev/null +++ b/examples/recordReader/XmlJavadocCommentsExtractor.xsl @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + +<!-- + Document : XmlJavadocCommentsExtractor.xsl + Created on : September 16, 2014, 11:30 AM + Description: + The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="1.0"> + <xsl:output method="xml" standalone="yes"/> + + <!-- copy xml by selecting only the following nodes, attributes and text --> + <xsl:template match="node()|text()|@*"> + <xsl:copy> + <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/> + </xsl:copy> + </xsl:template> + + <!-- Strip off the following paths from the selected xml --> + <xsl:template match="//root/package/interface/interface + |//root/package/interface/method/@qualified + |//root/package/class/interface + |//root/package/class/class + |//root/package/class/method/@qualified + |//root/package/class/field/@qualified" /> + + <xsl:strip-space elements="*"/> +</xsl:stylesheet> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/pom.xml ---------------------------------------------------------------------- diff --git a/examples/recordReader/pom.xml b/examples/recordReader/pom.xml new file mode 100644 index 0000000..007eaef --- /dev/null +++ b/examples/recordReader/pom.xml @@ -0,0 +1,297 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.example</groupId> + <version>1.0-SNAPSHOT</version> + <artifactId>recordReader</artifactId> + <packaging>jar</packaging> + + <name>File Record Reader</name> + <description>Simple application illustrating use of record reader operator</description> + + <properties> + <!-- change this if you desire to use a different version of Apex Core --> + <apex.version>3.5.0</apex.version> + <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath> + <malhar.version>3.6.0</malhar.version> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <version>2.9</version> + <configuration> + <downloadSources>true</downloadSources> + </configuration> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.3</version> + <configuration> + <encoding>UTF-8</encoding> + <source>1.7</source> + <target>1.7</target> + <debug>true</debug> + <optimize>false</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + </configuration> + </plugin> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/deps</outputDirectory> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <executions> + <execution> + <id>app-package-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${project.artifactId}-${project.version}-apexapp</finalName> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>src/assemble/appPackage.xml</descriptor> + </descriptors> + <archiverConfig> + <defaultDirectoryMode>0755</defaultDirectoryMode> + </archiverConfig> + <archive> + <manifestEntries> + <Class-Path>${apex.apppackage.classpath}</Class-Path> + <DT-Engine-Version>${apex.version}</DT-Engine-Version> + <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id> + <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name> + <DT-App-Package-Version>${project.version}</DT-App-Package-Version> + <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name> + <DT-App-Package-Description>${project.description}</DT-App-Package-Description> + </manifestEntries> + </archive> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <phase>package</phase> + <configuration> + <target> + <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar" + tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + <execution> + <!-- create resource directory for xml javadoc--> + <id>createJavadocDirectory</id> + <phase>generate-resources</phase> + <configuration> + <tasks> + <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/> + <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/> + </tasks> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9.1</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-${project.version}.apa</file> + <type>apa</type> + </artifact> + </artifacts> + <skipAttach>false</skipAttach> + </configuration> + </execution> + </executions> + </plugin> + + <!-- generate javdoc --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <!-- generate xml javadoc --> + <execution> + <id>xml-doclet</id> + <phase>generate-resources</phase> + <goals> + <goal>javadoc</goal> + </goals> + <configuration> + <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet> + <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam> + <useStandardDocletOptions>false</useStandardDocletOptions> + <docletArtifact> + <groupId>com.github.markusbernhardt</groupId> + <artifactId>xml-doclet</artifactId> + <version>1.0.4</version> + </docletArtifact> + </configuration> + </execution> + </executions> + </plugin> + <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags--> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>xml-maven-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>transform-xmljavadoc</id> + <phase>generate-resources</phase> + <goals> + <goal>transform</goal> + </goals> + </execution> + </executions> + <configuration> + <transformationSets> + <transformationSet> + <dir>${project.build.directory}/generated-resources/xml-javadoc</dir> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet> + <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir> + </transformationSet> + </transformationSets> + </configuration> + </plugin> + <!-- copy xml javadoc to class jar --> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>process-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/target/classes</outputDirectory> + <resources> + <resource> + <directory>${project.build.directory}/generated-resources/xml-javadoc</directory> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + + </build> + + <dependencies> + <!-- add your dependencies here --> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-library</artifactId> + <version>${malhar.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-contrib</artifactId> + <version>${malhar.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-contrib</artifactId> + <version>${malhar.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>net.sf.supercsv</groupId> + <artifactId>super-csv</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-common</artifactId> + <version>${apex.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.10</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-engine</artifactId> + <version>${apex.version}</version> + <scope>test</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/assemble/appPackage.xml ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/assemble/appPackage.xml b/examples/recordReader/src/assemble/appPackage.xml new file mode 100644 index 0000000..7ad071c --- /dev/null +++ b/examples/recordReader/src/assemble/appPackage.xml @@ -0,0 +1,43 @@ +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>appPackage</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/target/</directory> + <outputDirectory>/app</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/target/deps</directory> + <outputDirectory>/lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/site/conf</directory> + <outputDirectory>/conf</outputDirectory> + <includes> + <include>*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/META-INF</directory> + <outputDirectory>/META-INF</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/app</directory> + <outputDirectory>/app</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/resources</directory> + <outputDirectory>/resources</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/java/com/example/recordReader/Application.java ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/main/java/com/example/recordReader/Application.java b/examples/recordReader/src/main/java/com/example/recordReader/Application.java new file mode 100644 index 0000000..8b0dd75 --- /dev/null +++ b/examples/recordReader/src/main/java/com/example/recordReader/Application.java @@ -0,0 +1,32 @@ +/** + * Put your copyright and license info here. + */ +package com.example.recordReader; + +import org.apache.apex.malhar.lib.fs.FSRecordReaderModule; +import org.apache.apex.malhar.lib.fs.GenericFileOutputOperator.StringFileOutputOperator; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.DAG; +import com.datatorrent.api.StreamingApplication; +import com.datatorrent.api.annotation.ApplicationAnnotation; +import com.datatorrent.contrib.formatter.CsvFormatter; +import com.datatorrent.contrib.parser.CsvParser; + +@ApplicationAnnotation(name="RecordReaderExample") +public class Application implements StreamingApplication +{ + + @Override + public void populateDAG(DAG dag, Configuration conf) + { + FSRecordReaderModule recordReader = dag.addModule("recordReader", FSRecordReaderModule.class); + CsvParser csvParser = dag.addOperator("csvParser", CsvParser.class); + CsvFormatter formatter = dag.addOperator("formatter", new CsvFormatter()); + StringFileOutputOperator fileOutput = dag.addOperator("fileOutput", new StringFileOutputOperator()); + + dag.addStream("record", recordReader.records, csvParser.in); + dag.addStream("pojo", csvParser.out, formatter.in); + dag.addStream("string", formatter.out, fileOutput.input); + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java b/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java new file mode 100644 index 0000000..de20516 --- /dev/null +++ b/examples/recordReader/src/main/java/com/example/recordReader/TransactionsSchema.java @@ -0,0 +1,168 @@ +/** + * Copyright (c) 2016 DataTorrent, Inc. + * All rights reserved. + */ + +package com.example.recordReader; + +import java.util.Date; + +public class TransactionsSchema +{ + private String customerName; + private String customerPhone; + private String customerEmail; + private String city; + private String country; + private String uid; + private String accountNumber; + private long txId; + private Date txDate; + private double txAmount; + + public String getCustomerName() + { + return customerName; + } + + public void setCustomerName(String customerName) + { + this.customerName = customerName; + } + + public String getCustomerPhone() + { + return customerPhone; + } + + public void setCustomerPhone(String customerPhone) + { + this.customerPhone = customerPhone; + } + + public String getCustomerEmail() + { + return customerEmail; + } + + public void setCustomerEmail(String customerEmail) + { + this.customerEmail = customerEmail; + } + + public String getCity() + { + return city; + } + + public void setCity(String city) + { + this.city = city; + } + + public String getCountry() + { + return country; + } + + public void setCountry(String country) + { + this.country = country; + } + + public String getUid() + { + return uid; + } + + public void setUid(String uid) + { + this.uid = uid; + } + + public String getAccountNumber() + { + return accountNumber; + } + + public void setAccountNumber(String accountNumber) + { + this.accountNumber = accountNumber; + } + + public long getTxId() + { + return txId; + } + + public void setTxId(long txId) + { + this.txId = txId; + } + + public Date getTxDate() + { + return txDate; + } + + public void setTxDate(Date txDate) + { + this.txDate = txDate; + } + + public double getTxAmount() + { + return txAmount; + } + + public void setTxAmount(double txAmount) + { + this.txAmount = txAmount; + } + + @Override + public String toString() + { + return "TransactionsSchema [customerName=" + customerName + ", customerPhone=" + customerPhone + ", customerEmail=" + + customerEmail + ", city=" + city + ", country=" + country + ", uid=" + uid + ", accountNumber=" + + accountNumber + ", txId=" + txId + ", txDate=" + txDate + ", txAmount=" + txAmount + + "]\n"; + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = 1; + result = prime * result + ((txDate == null) ? 0 : txDate.hashCode()); + result = prime * result + (int)(txId ^ (txId >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TransactionsSchema other = (TransactionsSchema)obj; + if (txDate == null) { + if (other.txDate != null) { + return false; + } + } else if (!txDate.equals(other.txDate)) { + return false; + } + if (txId != other.txId) { + return false; + } + return true; + } + +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/resources/META-INF/CustomerTxnData ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/main/resources/META-INF/CustomerTxnData b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData new file mode 100644 index 0000000..88322ab --- /dev/null +++ b/examples/recordReader/src/main/resources/META-INF/CustomerTxnData @@ -0,0 +1,100 @@ +Desirae I. Hart|91-756-943-24728|[email protected]|Delhi|India|7F2BAD18-D1D1-8DB6-A519-70DA658EF6DE|1048.2446157601|245407|2016-06-17 21:19:02|56095.4691099477 +Fay B. Page|91-952-299-03140|[email protected]|Durg|India|5E8E12D3-34CC-53E1-7622-BC533542B930|939.9385330598|245408|2016-06-17 21:19:02|93013.7443360696 +Herrod F. Mcknight|91-609-146-21345|[email protected]|Haridwar|India|F589AF28-C10C-3EF1-E9DA-7CAAAD8E453D|1009.5714354316|245409|2016-06-17 21:19:03|57130.4933466919 +Suki K. Hutchinson|91-235-565-54483|[email protected]|Panjim|India|A263A773-E228-0C5B-D9CC-A5AE37A9F3F5|990.5143009573|245410|2016-06-17 21:19:03|88348.7049292828 +Tucker H. Strickland|91-353-281-78811|[email protected]|Aizwal|India|748AAD2B-8CB4-6010-2C80-F3B0FE107B4E|1027.2343235232|245411|2016-06-17 21:19:03|63602.53265577 +Carly L. Alvarado|91-364-259-91891|[email protected]|Mandya|India|292345E9-45E1-AE6C-9051-79681BE10E0A|964.9602866876|245412|2016-06-17 21:19:03|63777.4412787906 +Signe H. Carlson|91-736-760-21206|[email protected]|Basirhat|India|ABFCB28C-98B0-93E4-C5E9-D572648FE467|967.6980193409|245413|2016-06-17 21:19:04|81425.8673128708 +Timothy U. Cantrell|91-266-620-64253|[email protected]|Dibrugarh|India|46ED3B8F-5D19-D3FD-2833-7E6D956090D6|990.0113497158|245414|2016-06-17 21:19:04|20765.2379785903 +Todd M. Hunter|91-131-862-91105|[email protected]|Pondicherry|India|73DDE1E7-7B86-53A0-8296-8DDCC0A8022A|999.2001773265|245416|2016-06-17 21:19:04|28079.5354148245 +Martin J. Joyce|91-312-843-82341|[email protected]|Gandhidham|India|46247FA6-883D-2DAA-78E4-626F41937905|974.8521165154|245415|2016-06-17 21:19:04|98219.3968546873 +Cruz D. Slater|91-399-580-01355|[email protected]|Kalyan|India|7A0AE886-CDBF-98FF-1808-1A977B06F939|989.5735205715|245418|2016-06-17 21:19:05|69472.6529645815 +Keefe M. Hodge|91-123-766-63270|[email protected]|Bokaro Steel City|India|5488D1DE-E840-827F-2D45-347A58ECF72C|1059.6676733575|245417|2016-06-17 21:19:05|60538.9043471269 +Minerva R. Velez|91-550-563-14205|[email protected]|Bhusawal|India|C2EA3D6F-EEBE-7E7D-4DFD-DA90FAAAC04F|965.3100959637|245419|2016-06-17 21:19:05|22378.4665981816 +Gillian P. Bates|91-530-835-01656|[email protected]|Kohima|India|8882E4F0-0AB1-6098-EEF8-0230BBB931B2|978.3566576884|245420|2016-06-17 21:19:05|57058.528246018 +Jael I. Humphrey|91-815-502-83243|[email protected]|Chandigarh|India|0D8ADC96-B858-60A9-4F65-3BACADC808A8|1047.1902156238|245421|2016-06-17 21:19:06|71348.2114533201 +Eleanor D. Hart|91-351-706-26576|[email protected]|Imphal|India|6DE39307-4D0B-77B0-8DAF-36417261DA2B|1021.3673266291|245422|2016-06-17 21:19:06|60474.8184384689 +Kaitlin U. Higgins|91-738-770-10870|[email protected]|Kohima|India|C383AF31-636B-7897-6CF3-6B261BCDE018|1036.0644769672|245424|2016-06-17 21:19:06|48378.6503012718 +Giacomo A. Puckett|91-859-732-83951|[email protected]|Silchar|India|5E41A5A4-A4B3-17FE-F83D-6F6BF36B0F2D|979.2148681363|245423|2016-06-17 21:19:06|33445.0439158336 +Jamalia Z. Hensley|91-395-592-56855|[email protected]|Munger|India|7B98AA41-BC35-497E-6F2E-BBA815F84B28|1033.5638665941|245425|2016-06-17 21:19:07|63406.9037231582 +Raja L. Barrett|91-213-236-37863|[email protected]|Agartala|India|730F660A-4A6E-65B3-D64D-6A0AD4BD16F2|1051.51196186|245426|2016-06-17 21:19:07|16656.8948714763 +Audra D. Haley|91-909-551-98035|[email protected]|Shimla|India|5F6BD395-CFB1-86A3-232B-3B461C11B032|985.3099956934|245427|2016-06-17 21:19:07|42576.4326009901 +Burke G. Velez|91-480-634-47735|[email protected]|Shimla|India|C0D51785-79E5-86C1-2F60-6E2B4BC368E9|1014.3374004511|245428|2016-06-17 21:19:07|68099.3435062983 +Helen O. Watson|91-388-550-33596|[email protected]|Rewa|India|267EA362-E266-01FC-2371-065D4B150D6B|995.395922684|245429|2016-06-17 21:19:08|80594.4138667748 +Dillon E. Manning|91-412-483-18804|[email protected]|Kohima|India|08BD8752-4C34-1C2F-7EC5-063BE9C0025F|988.0527516287|245430|2016-06-17 21:19:08|34997.7855937537 +Darryl H. Preston|91-900-557-62018|[email protected]|Srinagar|India|BAE0CB80-33BC-23AD-4F23-6EED717A1FA9|1005.2310251378|245432|2016-06-17 21:19:08|90995.8552344935 +Teagan L. Merritt|91-755-577-05229|[email protected]|Daman|India|A65E89EC-BD39-F826-BF3A-2E4DFD1FFE96|1031.6923612578|245431|2016-06-17 21:19:08|76129.6754380442 +Blythe V. Hester|91-143-693-74169|[email protected]|Nagarcoil|India|457DA7DF-8473-FDA2-DFA8-9CA6FE70AF53|980.0187576543|245433|2016-06-17 21:19:09|1792.7823874316 +Felicia Z. Maynard|91-177-925-06452|[email protected]|Jammu|India|7511020D-252A-60E3-0259-B55174DB2BE0|962.3470002516|245434|2016-06-17 21:19:09|27442.1888130468 +Sopoline B. Swanson|91-512-190-23174|[email protected]|Agartala|India|92A84BDE-A231-715C-5B0A-A2C52FE4C726|1000.4777986036|245436|2016-06-17 21:19:09|32101.1072345695 +Christen T. Cabrera|91-956-276-72371|[email protected]|Tiruvannamalai|India|1B06945D-2595-5445-2694-12341158C45B|997.3852528283|245435|2016-06-17 21:19:09|86664.8353687926 +Chancellor C. Villarreal|91-882-782-21856|[email protected]|Pondicherry|India|33EBFDA7-8EEF-48A5-B323-AF7A7A57C4F9|984.5578096612|245437|2016-06-17 21:19:10|46584.2856419789 +Adara A. Price|91-334-227-26172|[email protected]|Shahjahanpur|India|82F22680-CD3B-F1E7-45F9-8E2ADAA5ED42|988.0619905918|245438|2016-06-17 21:19:10|21626.2511858243 +Dana B. Lucas|91-781-212-83146|[email protected]|Gangtok|India|88223C00-DCFA-F40B-AC4C-97FA79062789|976.7241552536|245440|2016-06-17 21:19:10|76481.528985468 +Alma X. Ingram|91-944-103-76290|[email protected]|Jammu|India|4C0C1266-DE7B-6FE7-6FFF-E0263987C15B|975.6210776138|245439|2016-06-17 21:19:10|64898.0024641107 +Cullen P. Glenn|91-388-717-53901|[email protected]|Kohima|India|BA6BF2F4-8F94-C4C2-740E-2A3B587C4B3A|1031.9092818916|245442|2016-06-17 21:19:11|9437.7993039509 +Darius S. Allen|91-978-355-75025|[email protected]|Trivandrum|India|D59B0CBB-A08D-DE3B-DE5F-426C0C96CC59|1032.6395532022|245441|2016-06-17 21:19:11|12454.7012410075 +Jordan J. Frye|91-399-558-07454|[email protected]|Chandigarh|India|EBCC5349-6EC4-356A-F897-20BFBF220087|1019.8537249814|245444|2016-06-17 21:19:11|32040.2035007451 +August N. Chase|91-447-831-17890|[email protected]|Agartala|India|8871CD66-C471-B709-61C4-DA8E11395864|1016.3890473484|245443|2016-06-17 21:19:11|45318.9872586436 +Teegan Z. House|91-834-850-62568|[email protected]|Aizwal|India|13B85BBD-EA23-70B2-1723-9F1D6046B42E|959.962281601|245445|2016-06-17 21:19:12|88784.9082302192 +Aquila B. Moran|91-851-475-57283|[email protected]|Thalassery|India|823590C6-3381-1E11-8303-C3DA2A0E1821|993.9503426924|245446|2016-06-17 21:19:12|85906.741529358 +Dominic T. Dalton|91-504-884-93067|[email protected]|Rohtak|India|4EC579B1-4095-421D-27D5-95BD94551B81|988.6711045096|245448|2016-06-17 21:19:12|96034.4758503845 +Francis C. Gay|91-155-131-31144|[email protected]|Panjim|India|BEA0E8B8-2153-55C3-9FAC-B645F75DC11F|1041.9520701363|245447|2016-06-17 21:19:12|28285.6913337028 +Florence K. Cox|91-509-817-87497|[email protected]|Bhilai|India|C613A6CF-0DDF-457B-1611-048AF476CD9F|984.7873386645|245449|2016-06-17 21:19:13|8045.8599610703 +Desirae F. Mooney|91-146-543-19767|[email protected]|Gangtok|India|58A86FB7-1F20-2C33-2682-9B20165B5CF9|965.4688425994|245450|2016-06-17 21:19:13|25829.5686136197 +Virginia O. Welch|91-338-235-55295|[email protected]|Srinagar|India|350C6D1C-8C2D-BE4A-EAB5-8098C2BC8054|981.8861112513|245451|2016-06-17 21:19:13|55435.8774902126 +Simon P. Estes|91-101-148-34019|[email protected]|Kavaratti|India|D42E0FB2-6093-FD6F-EA2E-DC470647340F|961.8009052379|245452|2016-06-17 21:19:13|26959.0061609105 +Garth M. Fuentes|91-401-871-57104|[email protected]|Vejalpur|India|1450A844-53D1-6315-4415-5C75B08FC228|974.9367040591|245454|2016-06-17 21:19:14|93043.0249827518 +Jaquelyn D. Rodgers|91-913-783-42186|[email protected]|Nagpur|India|D6CE7D75-0659-454A-40DF-3AF296DD4C58|976.7352615741|245453|2016-06-17 21:19:14|59609.6850497366 +Kasper F. Hatfield|91-632-379-95960|[email protected]|Korba|India|D59B8BE9-DA20-7B57-9942-2C15BCF28603|1000.4924219911|245456|2016-06-17 21:19:14|3275.3243581371 +Isaiah A. Warner|91-626-859-53011|[email protected]|Daman|India|20B5FE0D-0965-1E2D-681B-5E688DA1CA65|1001.1159994583|245455|2016-06-17 21:19:14|76836.6239820194 +Keaton Y. Solis|91-301-918-77318|[email protected]|Rajkot|India|16CFA137-3FD3-8D3D-7607-54577A6A18AB|987.5936149956|245458|2016-06-17 21:19:15|89465.3972332774 +Regan D. Underwood|91-869-304-48740|[email protected]|Daman|India|241FD5E5-01D8-7BEA-BE3A-3435B1D6577F|967.2325392222|245457|2016-06-17 21:19:15|81430.0742569319 +Laurel K. Hardy|91-535-589-44540|[email protected]|Shimla|India|46689123-0B18-3A05-2E9D-1995EABE674F|947.0982666227|245459|2016-06-17 21:19:15|14451.1023722487 +Vielka J. Roberson|91-545-218-53793|[email protected]|Srinagar|India|E6384D6E-2AC4-3AF7-8C1C-A37DD4B013C6|933.9068440201|245460|2016-06-17 21:19:15|75600.2030709746 +Brynn S. Morrison|91-977-938-06240|[email protected]|Daman|India|859E9B35-3A21-F7FD-A9A5-167F7FE6A7F5|1002.7243083243|245461|2016-06-17 21:19:16|19882.1569818146 +Graiden T. Jones|91-868-733-90988|[email protected]|Nellore|India|C133E546-1E36-A6EF-9980-BB635F247290|973.7286142544|245462|2016-06-17 21:19:16|4393.9931397479 +Tanya X. Dominguez|91-553-110-11147|[email protected]|Anantapur|India|DEF2CE91-CAF2-88BA-C50B-AB7EA1D73C63|959.1678617895|245464|2016-06-17 21:19:16|42536.8767788127 +Anne D. Pace|91-188-230-70605|[email protected]|Agartala|India|32FC1F14-21F7-E940-7C9D-B80F44EE4063|1021.5754177485|245463|2016-06-17 21:19:16|32755.1536052658 +Hanna I. Sellers|91-322-901-42448|[email protected]|Barddhaman|India|3D4F2FBC-FFCE-EE5C-5D2C-C60856D33B7D|993.2866819089|245466|2016-06-17 21:19:17|75662.1622559809 +Christine R. Gross|91-380-558-08928|[email protected]|Shillong|India|DD81CFAC-DE87-E8A1-2850-E2637912543C|951.970651195|245465|2016-06-17 21:19:17|80146.9990770126 +Kirby A. Morse|91-299-538-45635|[email protected]|Itanagar|India|CFF08336-B5C9-EECC-1F2D-07ADE42587FE|1009.5054026816|245467|2016-06-17 21:19:17|72602.1251286646 +Ryder B. Vega|91-320-455-77145|[email protected]|Gondiya|India|15F3478F-4D7E-562C-E189-945E655641E9|1006.8962892921|245468|2016-06-17 21:19:17|8089.8058610084 +Demetrius Q. Pope|91-179-308-28350|[email protected]|Imphal|India|CBF9C119-3E02-13F0-842A-BAF0AB46382D|1042.9498091927|245470|2016-06-17 21:19:18|83959.394047181 +Bryar M. Hendricks|91-617-948-40647|[email protected]|Kanchrapara|India|64188EFF-B7B1-AF75-EA3D-013CCA945F81|968.5880217334|245469|2016-06-17 21:19:18|85996.1671440447 +Ann P. Mann|91-395-300-37963|[email protected]|Imphal|India|12818782-A86C-63A2-AB08-A5F281DA954B|937.3545836376|245472|2016-06-17 21:19:18|83236.1732288256 +Tyrone N. Bender|91-515-367-84014|[email protected]|Chandigarh|India|D5109D0E-7DB6-A084-F0F9-E2BEF2B10E4A|1013.1764242663|245471|2016-06-17 21:19:18|36038.3036732134 +Miranda W. Sykes|91-774-614-92485|[email protected]|Gangtok|India|6EC7240D-5331-5F34-D1B8-38E2024A1925|992.518070982|245473|2016-06-17 21:19:19|97197.8457445721 +Ralph F. Walker|91-815-285-72212|[email protected]|Farrukhabad-cum-Fatehgarh|India|B7982CE7-B815-2794-504B-A8C3ECF0B592|992.3386856766|245474|2016-06-17 21:19:19|60698.1579345159 +Kelsie J. Mack|91-112-975-50329|[email protected]|Hazaribag|India|5CACD398-932B-449D-80AC-5302AE35E027|1027.0394067971|245475|2016-06-17 21:19:19|48589.914064424 +Zenia F. Bond|91-445-745-77345|[email protected]|Port Blair|India|68EDFE31-8E62-FF3B-3FBA-49B436D7142C|996.5071564618|245476|2016-06-17 21:19:19|40633.7414556007 +Todd V. Barton|91-312-429-52379|[email protected]|Dehradun|India|7D22103A-1DA0-2E4E-BFD7-8117133A983A|977.4969081079|245477|2016-06-17 21:19:20|49497.7649558266 +Kerry O. Franco|91-802-106-72427|[email protected]|Kohima|India|FC6049B3-9FD4-9A53-1A7E-C50394D4F4CD|1004.0387939017|245478|2016-06-17 21:19:20|43754.1503070853 +April I. Clarke|91-785-400-08200|[email protected]|Daman|India|363E468F-BD44-6607-CDD6-226A6FB48B89|990.0985191984|245479|2016-06-17 21:19:20|90752.7220658782 +Randall B. Workman|91-722-286-44955|[email protected]|Gujrat|India|1A402674-FD62-8F1B-7AE1-CFAA7E85006A|1012.6352839308|245480|2016-06-17 21:19:20|1815.8797213967 +Blaine J. Martinez|91-987-901-22060|[email protected]|Kohima|India|7FDD906D-AB41-89EC-D546-B1E04D31FE49|1020.339672989|245481|2016-06-17 21:19:21|51409.0038738681 +May B. Schroeder|91-123-334-23814|[email protected]|Orai|India|2E83862F-C3A4-4A4B-4D9C-C60AC4435A04|982.3424542846|245482|2016-06-17 21:19:21|89159.6292602544 +Whitney N. Tran|91-923-716-39943|[email protected]|Purulia|India|FEE99492-4B48-533F-EB36-5EA3528A121C|1007.9029941609|245483|2016-06-17 21:19:21|24692.473001632 +Ingrid A. Head|91-596-380-43300|[email protected]|Port Blair|India|735007BF-1F13-95E9-7C3E-536C0452714D|995.4883512745|245484|2016-06-17 21:19:21|54890.8030672057 +Berk E. Gould|91-110-540-75246|[email protected]|Alwar|India|FB0297F7-8BC1-86F0-8CCD-C01538B7E5BE|984.2152917929|245486|2016-06-17 21:19:22|70311.7302535036 +Meredith G. Parks|91-154-792-20142|[email protected]|Chandigarh|India|CD2691BE-ADB4-87DF-CA3B-88512330E745|960.8832140455|245485|2016-06-17 21:19:22|95293.2215958629 +Cora K. Carrillo|91-786-268-62522|[email protected]|Dindigul|India|CAA843B7-F7C8-AD16-3F05-89DADB6EB288|1001.8761512414|245488|2016-06-17 21:19:22|59127.2684732433 +Galena D. Farmer|91-951-409-02162|[email protected]|Shillong|India|A46AF538-4BEF-D24F-870F-DD4CD20F4DC6|1022.2222771884|245487|2016-06-17 21:19:22|36667.9833050979 +Mollie E. Snow|91-816-823-87138|[email protected]|Tiruvannamalai|India|A66AFB86-5297-61AC-2A2A-87C23F0E96E1|966.8626148916|245490|2016-06-17 21:19:23|28859.7500576673 +Abra E. Mccray|91-100-997-13066|[email protected]|Delhi|India|E49C0B30-586F-ECAA-FCBD-8F2898AA7771|1043.6824086302|245489|2016-06-17 21:19:23|79521.2870470628 +Andrew I. Lowe|91-859-767-29836|[email protected]|Aizwal|India|A928DE5C-6C18-9B9B-C4F0-3165F2E53C19|955.4198061291|245491|2016-06-17 21:19:23|34946.6776395577 +Zachery B. Marshall|91-103-978-47466|[email protected]|Vijayawada|India|E0D79F5F-2CB6-B9FC-B917-2D627141423B|969.8127419963|245492|2016-06-17 21:19:23|74671.8685208157 +Miranda M. Madden|91-310-815-30233|[email protected]|Brahmapur|India|55E04315-DE60-1C1D-78D6-878C9488BA06|963.5417067954|245493|2016-06-17 21:19:24|42089.8687934832 +Ian N. Johnson|91-651-594-79593|[email protected]|Agartala|India|EDBCCD62-4871-4938-60C5-0CD3023CF333|1012.7053923589|245494|2016-06-17 21:19:24|97376.7240252239 +Macey V. Hinton|91-358-832-49086|[email protected]|Sambalpur|India|49E4842E-0EB8-D48A-C1EC-A3F0FBAC92B4|982.9530728621|245496|2016-06-17 21:19:24|61850.2465402803 +Talon M. Calhoun|91-280-397-90948|[email protected]|Pallavaram|India|E743767A-B1FF-C71C-B0D4-4C106D65AE3B|1040.9448702398|245495|2016-06-17 21:19:24|49928.6243188531 +Dillon A. Reese|91-900-545-84875|[email protected]|Itanagar|India|9429B72A-36AA-8C78-2988-7FF48315094A|1029.5299833946|245498|2016-06-17 21:19:25|86990.7236411972 +Boris E. Boyd|91-480-944-74996|[email protected]|Panjim|India|37962E2E-4B3D-27DD-1213-4FF1B5551CB4|969.407095963|245497|2016-06-17 21:19:25|32987.5284224486 +Inez U. Higgins|91-258-514-16019|[email protected]|Itanagar|India|D48C171C-A009-3813-5951-50A526BE6B58|1005.0207448325|245500|2016-06-17 21:19:25|68069.3550977988 +Leroy N. Snyder|91-682-436-52541|[email protected]|Itanagar|India|BE155410-1F6D-AC60-C0B9-290E9426D5CF|993.1732235668|245499|2016-06-17 21:19:25|64376.3818134466 +George M. Case|91-760-901-60977|[email protected]|Pondicherry|India|B924E744-29E3-8B6E-3EE9-A48A3BC9072B|987.476080645|245501|2016-06-17 21:19:26|97859.0229750841 +Kane E. Simon|91-461-207-64947|[email protected]|Haridwar|India|13C1AD7F-48D5-C6E5-892B-0345E7E070DB|983.5254219297|245502|2016-06-17 21:19:26|21912.3442110069 +Wanda U. Talley|91-398-803-35370|[email protected]|Gangtok|India|5E2ED32D-621C-0ACE-71B6-F2F9746DD0C9|994.6344833977|245503|2016-06-17 21:19:26|91796.2772584239 +Asher U. Mccullough|91-149-955-32468|[email protected]|Itanagar|India|3FA3514B-6974-06E3-EF7B-4841487A40D2|985.0811909206|245504|2016-06-17 21:19:26|71603.8572439688 +Malik E. Keller|91-391-507-50089|[email protected]|Daman|India|05CDC866-65A3-E55C-EEE8-C73217F2071A|986.9798931399|245505|2016-06-17 21:19:27|65100.1941324091 +Yasir V. Howell|91-901-430-36468|[email protected]|Mandya|India|F90D3C87-4C53-A5E9-D3F8-692C068CBCED|997.1935636906|245506|2016-06-17 21:19:27|35511.2898055096 http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/main/resources/META-INF/properties.xml ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/main/resources/META-INF/properties.xml b/examples/recordReader/src/main/resources/META-INF/properties.xml new file mode 100644 index 0000000..06961f1 --- /dev/null +++ b/examples/recordReader/src/main/resources/META-INF/properties.xml @@ -0,0 +1,159 @@ +<?xml version="1.0"?> +<configuration> + <!-- + <property> + <name>dt.application.{appName}.operator.{opName}.prop.{propName}</name> + <value>some-default-value (if value is not specified, it is required from the user or custom config when launching)</value> + </property> + --> + <!-- memory assigned to app master + <property> + <name>dt.attr.MASTER_MEMORY_MB</name> + <value>1024</value> + </property> + --> + <property> + <name>dt.application.RecordReaderExample.operator.recordReader.prop.files</name> + <value>src/main/resources/META-INF/CustomerTxnData</value> + </property> + <property> + <name>dt.application.RecordReaderExample.attr.CHECKPOINT_WINDOW_COUNT</name> + <value>10</value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.csvParser.prop.schema</name> + <value>{ + "separator": "|", + "quoteChar": "\"", + "fields": [ + { + "name": "customerName", + "type": "String" + }, + { + "name": "customerPhone", + "type": "String" + }, + { + "name": "customerEmail", + "type": "String" + }, + { + "name": "city", + "type": "String" + }, + { + "name": "country", + "type": "String" + }, + { + "name": "uid", + "type": "String" + }, + { + "name": "accountNumber", + "type": "String" + }, + { + "name": "txId", + "type": "long" + }, + { + "name": "txDate", + "type": "Date", + "constraints" : { + "format" : "yyyy-MM-dd HH:mm:ss" + } + }, + { + "name": "txAmount", + "type": "double" + } + ] + } + </value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.formatter.prop.schema</name> + <value>{ + "separator": "|", + "quoteChar": "\"", + "lineDelimiter": "", + "fields": [ + { + "name": "customerName", + "type": "String" + }, + { + "name": "customerPhone", + "type": "String" + }, + { + "name": "customerEmail", + "type": "String" + }, + { + "name": "city", + "type": "String" + }, + { + "name": "country", + "type": "String" + }, + { + "name": "uid", + "type": "String" + }, + { + "name": "accountNumber", + "type": "String" + }, + { + "name": "txId", + "type": "long" + }, + { + "name": "txDate", + "type": "Date", + "constraints" : { + "format" : "yyyy-MM-dd HH:mm:ss" + } + }, + { + "name": "txAmount", + "type": "double" + } + ] + } + </value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.csvParser.port.out.attr.TUPLE_CLASS</name> + <value>com.example.recordReader.TransactionsSchema</value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.formatter.port.in.attr.TUPLE_CLASS</name> + <value>com.example.recordReader.TransactionsSchema</value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.fileOutput.prop.filePath</name> + <value>/tmp</value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.fileOutput.prop.outputFileName</name> + <value>output.txt</value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.fileOutput.prop.tupleSeparator</name> + <value></value> + </property> + <property> + <name>dt.application.RecordReaderExample.operator.fileOutput.prop.maxIdleWindows</name> + <value>2</value> + </property> + <property> + <name>dt.loggers.level</name> + <value>com.datatorrent.*:DEBUG,org.apache.apex.malhar.*:DEBUG</value> + </property> +</configuration> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java b/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java new file mode 100644 index 0000000..222ad06 --- /dev/null +++ b/examples/recordReader/src/test/java/com/example/recordReader/ApplicationTest.java @@ -0,0 +1,91 @@ +package com.example.recordReader; + +import java.io.File; +import java.io.IOException; + +import javax.validation.ConstraintViolationException; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.LocalMode; + +/** + * Test application in local mode. + */ +public class ApplicationTest +{ + private String outputDir; + + public static class TestMeta extends TestWatcher + { + public String baseDirectory; + + @Override + protected void starting(org.junit.runner.Description description) + { + this.baseDirectory = "target/" + description.getClassName() + "/" + description.getMethodName(); + } + + @Override + protected void finished(Description description) + { + super.finished(description); + try { + FileUtils.forceDelete(new File(baseDirectory)); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + @Rule + public TestMeta testMeta = new TestMeta(); + + @Before + public void setup() throws Exception + { + outputDir = testMeta.baseDirectory + File.separator + "output"; + } + + @Test + public void testApplication() throws IOException, Exception + { + try { + LocalMode lma = LocalMode.newInstance(); + Configuration conf = new Configuration(false); + conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml")); + conf.set("dt.application.RecordReaderExample.operator.fileOutput.prop.filePath", outputDir); + File outputfile = FileUtils.getFile(outputDir, "output.txt_5.0"); + + lma.prepareDAG(new Application(), conf); + LocalMode.Controller lc = lma.getController(); + lc.runAsync(); + + // wait for tuples to show up + while (!outputfile.exists()) { + System.out.println("Sleeping ...."); + Thread.sleep(1000); + } + + lc.shutdown(); + Assert.assertTrue( + FileUtils.contentEquals( + FileUtils.getFile( + conf.get("dt.application.RecordReaderExample.operator.recordReader.prop.files") + ),outputfile)); + + } catch (ConstraintViolationException e) { + Assert.fail("constraint violations: " + e.getConstraintViolations()); + } + } + +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/11f37671/examples/recordReader/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/examples/recordReader/src/test/resources/log4j.properties b/examples/recordReader/src/test/resources/log4j.properties new file mode 100644 index 0000000..3bfcdc5 --- /dev/null +++ b/examples/recordReader/src/test/resources/log4j.properties @@ -0,0 +1,21 @@ +log4j.rootLogger=DEBUG,CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.RFA.File=/tmp/app.log + +# to enable, add SYSLOG to rootLogger +log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender +log4j.appender.SYSLOG.syslogHost=127.0.0.1 +log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout +log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n +log4j.appender.SYSLOG.Facility=LOCAL1 + +log4j.logger.org=info +#log4j.logger.org.apache.commons.beanutils=warn +log4j.logger.com.datatorrent=debug
