SPOI-8652 Example for Enricher
Project: http://git-wip-us.apache.org/repos/asf/apex-malhar/repo Commit: http://git-wip-us.apache.org/repos/asf/apex-malhar/commit/dd509f2a Tree: http://git-wip-us.apache.org/repos/asf/apex-malhar/tree/dd509f2a Diff: http://git-wip-us.apache.org/repos/asf/apex-malhar/diff/dd509f2a Branch: refs/heads/master Commit: dd509f2a17b4139b335949c305566a5ca48ee8e0 Parents: 4573528 Author: Chinmay <[email protected]> Authored: Wed Jul 20 20:35:01 2016 +0530 Committer: Lakshmi Prasanna Velineni <[email protected]> Committed: Sun Mar 26 11:43:48 2017 -0700 ---------------------------------------------------------------------- examples/enricher/README.md | 13 + .../enricher/XmlJavadocCommentsExtractor.xsl | 44 +++ examples/enricher/pom.xml | 308 +++++++++++++++++++ examples/enricher/src/assemble/appPackage.xml | 43 +++ .../com/example/myapexapp/DataGenerator.java | 94 ++++++ .../myapexapp/EnricherAppWithJSONFile.java | 47 +++ .../example/myapexapp/LineOutputOperator.java | 34 ++ .../main/java/com/example/myapexapp/POJO.java | 49 +++ .../com/example/myapexapp/POJOEnriched.java | 71 +++++ .../src/main/resources/META-INF/properties.xml | 41 +++ .../src/main/resources/circleMapping.txt | 10 + .../com/example/myapexapp/ApplicationTest.java | 31 ++ .../src/test/resources/log4j.properties | 21 ++ 13 files changed, 806 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/README.md ---------------------------------------------------------------------- diff --git a/examples/enricher/README.md b/examples/enricher/README.md new file mode 100644 index 0000000..a987a0b --- /dev/null +++ b/examples/enricher/README.md @@ -0,0 +1,13 @@ +This sample application show how to use POJOEnricher to enrich streaming data using +external source. +The operators in order as as follows: +1. Random data generator which emits data in JSON string format +2. JSON Parser which takes JSON string and emits POJO +3. POJO Enricher which enriches input using file and emits output POJO +4. Line Output Operator which emits line by line output to File System +The output files start out with a `.tmp` extension and get renamed when they reach the +size bound. + +Similar to FSLoader JDBCLoader can be used when JDBC backend is required. + + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/XmlJavadocCommentsExtractor.xsl ---------------------------------------------------------------------- diff --git a/examples/enricher/XmlJavadocCommentsExtractor.xsl b/examples/enricher/XmlJavadocCommentsExtractor.xsl new file mode 100644 index 0000000..08075a9 --- /dev/null +++ b/examples/enricher/XmlJavadocCommentsExtractor.xsl @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> + +<!-- + Document : XmlJavadocCommentsExtractor.xsl + Created on : September 16, 2014, 11:30 AM + Description: + The transformation strips off all information except for comments and tags from xml javadoc generated by xml-doclet. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="1.0"> + <xsl:output method="xml" standalone="yes"/> + + <!-- copy xml by selecting only the following nodes, attributes and text --> + <xsl:template match="node()|text()|@*"> + <xsl:copy> + <xsl:apply-templates select="root|package|class|interface|method|field|type|comment|tag|text()|@name|@qualified|@text"/> + </xsl:copy> + </xsl:template> + + <!-- Strip off the following paths from the selected xml --> + <xsl:template match="//root/package/interface/interface + |//root/package/interface/method/@qualified + |//root/package/class/interface + |//root/package/class/class + |//root/package/class/method/@qualified + |//root/package/class/field/@qualified" /> + + <xsl:strip-space elements="*"/> +</xsl:stylesheet> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/pom.xml ---------------------------------------------------------------------- diff --git a/examples/enricher/pom.xml b/examples/enricher/pom.xml new file mode 100644 index 0000000..a93bcf5 --- /dev/null +++ b/examples/enricher/pom.xml @@ -0,0 +1,308 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>com.example</groupId> + <version>1.0-SNAPSHOT</version> + <artifactId>enricher</artifactId> + <packaging>jar</packaging> + + <!-- change these to the appropriate values --> + <name>Enricher</name> + <description>Example Use of POJO Enricher</description> + + <properties> + <!-- change this if you desire to use a different version of Apex Core --> + <apex.version>3.5.0</apex.version> + <apex.apppackage.classpath>lib/*.jar</apex.apppackage.classpath> + <malhar.version>3.6.0</malhar.version> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <version>2.9</version> + <configuration> + <downloadSources>true</downloadSources> + </configuration> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.3</version> + <configuration> + <encoding>UTF-8</encoding> + <source>1.7</source> + <target>1.7</target> + <debug>true</debug> + <optimize>false</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + </configuration> + </plugin> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>target/deps</outputDirectory> + <includeScope>runtime</includeScope> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <executions> + <execution> + <id>app-package-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${project.artifactId}-${project.version}-apexapp</finalName> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>src/assemble/appPackage.xml</descriptor> + </descriptors> + <archiverConfig> + <defaultDirectoryMode>0755</defaultDirectoryMode> + </archiverConfig> + <archive> + <manifestEntries> + <Class-Path>${apex.apppackage.classpath}</Class-Path> + <DT-Engine-Version>${apex.version}</DT-Engine-Version> + <DT-App-Package-Group-Id>${project.groupId}</DT-App-Package-Group-Id> + <DT-App-Package-Name>${project.artifactId}</DT-App-Package-Name> + <DT-App-Package-Version>${project.version}</DT-App-Package-Version> + <DT-App-Package-Display-Name>${project.name}</DT-App-Package-Display-Name> + <DT-App-Package-Description>${project.description}</DT-App-Package-Description> + </manifestEntries> + </archive> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <phase>package</phase> + <configuration> + <target> + <move file="${project.build.directory}/${project.artifactId}-${project.version}-apexapp.jar" + tofile="${project.build.directory}/${project.artifactId}-${project.version}.apa" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + <execution> + <!-- create resource directory for xml javadoc--> + <id>createJavadocDirectory</id> + <phase>generate-resources</phase> + <configuration> + <tasks> + <delete dir="${project.build.directory}/generated-resources/xml-javadoc"/> + <mkdir dir="${project.build.directory}/generated-resources/xml-javadoc"/> + </tasks> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.9.1</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>target/${project.artifactId}-${project.version}.apa</file> + <type>apa</type> + </artifact> + </artifacts> + <skipAttach>false</skipAttach> + </configuration> + </execution> + </executions> + </plugin> + + <!-- generate javdoc --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <!-- generate xml javadoc --> + <execution> + <id>xml-doclet</id> + <phase>generate-resources</phase> + <goals> + <goal>javadoc</goal> + </goals> + <configuration> + <doclet>com.github.markusbernhardt.xmldoclet.XmlDoclet</doclet> + <additionalparam>-d ${project.build.directory}/generated-resources/xml-javadoc -filename ${project.artifactId}-${project.version}-javadoc.xml</additionalparam> + <useStandardDocletOptions>false</useStandardDocletOptions> + <docletArtifact> + <groupId>com.github.markusbernhardt</groupId> + <artifactId>xml-doclet</artifactId> + <version>1.0.4</version> + </docletArtifact> + </configuration> + </execution> + </executions> + </plugin> + <!-- Transform xml javadoc to stripped down version containing only class/interface comments and tags--> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>xml-maven-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>transform-xmljavadoc</id> + <phase>generate-resources</phase> + <goals> + <goal>transform</goal> + </goals> + </execution> + </executions> + <configuration> + <transformationSets> + <transformationSet> + <dir>${project.build.directory}/generated-resources/xml-javadoc</dir> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <stylesheet>XmlJavadocCommentsExtractor.xsl</stylesheet> + <outputDir>${project.build.directory}/generated-resources/xml-javadoc</outputDir> + </transformationSet> + </transformationSets> + </configuration> + </plugin> + <!-- copy xml javadoc to class jar --> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>process-resources</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/target/classes</outputDirectory> + <resources> + <resource> + <directory>${project.build.directory}/generated-resources/xml-javadoc</directory> + <includes> + <include>${project.artifactId}-${project.version}-javadoc.xml</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + + </plugins> + + </build> + + <dependencies> + <!-- add your dependencies here --> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-library</artifactId> + <version>${malhar.version}</version> + <!-- + If you know that your application does not need transitive dependencies pulled in by malhar-library, + uncomment the following to reduce the size of your app package. + --> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>malhar-contrib</artifactId> + <version>${malhar.version}</version> + <!-- + If you know that your application does not need transitive dependencies pulled in by malhar-library, + uncomment the following to reduce the size of your app package. + --> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-common</artifactId> + <version>${apex.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.10</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.apex</groupId> + <artifactId>apex-engine</artifactId> + <version>${apex.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.github.fge</groupId> + <artifactId>json-schema-validator</artifactId> + <version>2.0.1</version> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>commons-compiler</artifactId> + <version>2.7.8</version> + <type>jar</type> + </dependency> + <dependency> + <groupId>org.codehaus.janino</groupId> + <artifactId>janino</artifactId> + <version>2.7.8</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/assemble/appPackage.xml ---------------------------------------------------------------------- diff --git a/examples/enricher/src/assemble/appPackage.xml b/examples/enricher/src/assemble/appPackage.xml new file mode 100644 index 0000000..7ad071c --- /dev/null +++ b/examples/enricher/src/assemble/appPackage.xml @@ -0,0 +1,43 @@ +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> + <id>appPackage</id> + <formats> + <format>jar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + <fileSets> + <fileSet> + <directory>${basedir}/target/</directory> + <outputDirectory>/app</outputDirectory> + <includes> + <include>${project.artifactId}-${project.version}.jar</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/target/deps</directory> + <outputDirectory>/lib</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/site/conf</directory> + <outputDirectory>/conf</outputDirectory> + <includes> + <include>*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/META-INF</directory> + <outputDirectory>/META-INF</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/app</directory> + <outputDirectory>/app</outputDirectory> + </fileSet> + <fileSet> + <directory>${basedir}/src/main/resources/resources</directory> + <outputDirectory>/resources</outputDirectory> + </fileSet> + </fileSets> + +</assembly> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java b/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java new file mode 100644 index 0000000..3afbb87 --- /dev/null +++ b/examples/enricher/src/main/java/com/example/myapexapp/DataGenerator.java @@ -0,0 +1,94 @@ +package com.example.myapexapp; + +import java.util.Random; + +import com.datatorrent.api.Context.OperatorContext; +import com.datatorrent.api.DefaultOutputPort; +import com.datatorrent.api.InputOperator; +import com.datatorrent.common.util.BaseOperator; + +/** + * Generates Subscriber Data: + * A Party Phone + * A Party IMEI + * A Party IMSI + * Circle Id + */ +public class DataGenerator extends BaseOperator implements InputOperator +{ + public static int NUM_CIRCLES = 10; + + private Random r; + private int count = 0; + private int limit = 1000; + + public final transient DefaultOutputPort<byte[]> output = new DefaultOutputPort<>(); + + @Override + public void setup(OperatorContext context) + { + r = new Random(System.currentTimeMillis()); + } + + @Override + public void beginWindow(long windowId) { + super.beginWindow(windowId); + count = 0; + } + + @Override + public void emitTuples() + { + if(count++ < limit) { + output.emit(getRecord()); + } + } + + private byte[] getRecord() + { + String phone = getRandomNumber(10); + String imsi = getHashInRange(phone, 15); + String imei = getHashInRange(imsi, 15); + String circleId = Math.abs(phone.hashCode()) % NUM_CIRCLES + ""; +// String record = MessageFormat.format(baseDataTemplate, phone, imsi, imei, circleId); + String record = "{" + + "\"phone\":\"" + phone + "\"," + + "\"imei\":\"" + imei+ "\"," + + "\"imsi\":\"" + imsi+ "\"," + + "\"circleId\":" + circleId + + "}"; + return record.getBytes(); + } + + private String getRandomNumber(int numDigits) + { + String retVal = (r.nextInt((9 - 1) + 1) + 1) + ""; + + for (int i = 0; i < numDigits - 1; i++) { + retVal += (r.nextInt((9 - 0) + 1) + 0); + } + return retVal; + } + + private String getHashInRange(String s, int n) + { + StringBuilder retVal = new StringBuilder(); + for (int i = 0, j = 0; i < n && j < s.length(); i++, j++) { + retVal.append(Math.abs(s.charAt(j) + "".hashCode()) % 10); + if (j == s.length() - 1) { + j = -1; + } + } + return retVal.toString(); + } + + public int getLimit() + { + return limit; + } + + public void setLimit(int limit) + { + this.limit = limit; + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java b/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java new file mode 100644 index 0000000..a0dab64 --- /dev/null +++ b/examples/enricher/src/main/java/com/example/myapexapp/EnricherAppWithJSONFile.java @@ -0,0 +1,47 @@ +package com.example.myapexapp; + +import java.util.ArrayList; + +import com.datatorrent.contrib.enrich.JsonFSLoader; +import com.datatorrent.contrib.enrich.POJOEnricher; +import com.datatorrent.contrib.parser.JsonParser; +import com.datatorrent.lib.io.ConsoleOutputOperator; +import org.apache.hadoop.conf.Configuration; + +import com.datatorrent.api.DAG; +import com.datatorrent.api.StreamingApplication; +import com.datatorrent.api.annotation.ApplicationAnnotation; + +@ApplicationAnnotation(name="EnricherAppWithJSONFile") +public class EnricherAppWithJSONFile implements StreamingApplication +{ + @Override + public void populateDAG(DAG dag, Configuration conf) + { + DataGenerator dataGenerator = dag.addOperator("DataGenerator", DataGenerator.class); + JsonParser parser = dag.addOperator("Parser", JsonParser.class); + + /** + * FSLoader is used to configure Enricher backend. Property of FSLoader file which is fileName is set in + * properties.xml file. + * The format that is used to read the file is present as an example in resources/circleMapping.txt file. + */ + JsonFSLoader fsLoader = new JsonFSLoader(); + POJOEnricher enrich = dag.addOperator("Enrich", POJOEnricher.class); + enrich.setStore(fsLoader); + + ArrayList includeFields = new ArrayList(); + includeFields.add("circleName"); + ArrayList lookupFields = new ArrayList(); + lookupFields.add("circleId"); + + enrich.setIncludeFields(includeFields); + enrich.setLookupFields(lookupFields); + + ConsoleOutputOperator console = dag.addOperator("Console", ConsoleOutputOperator.class); + + dag.addStream("Parse", dataGenerator.output, parser.in); + dag.addStream("Enrich", parser.out, enrich.input); + dag.addStream("Console", enrich.output, console.input); + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java b/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java new file mode 100644 index 0000000..3b7a298 --- /dev/null +++ b/examples/enricher/src/main/java/com/example/myapexapp/LineOutputOperator.java @@ -0,0 +1,34 @@ +package com.example.myapexapp; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import javax.validation.constraints.NotNull; + +import com.datatorrent.lib.io.fs.AbstractFileOutputOperator; + +/** + * Converts each tuple to a string and writes it as a new line to the output file + */ +public class LineOutputOperator extends AbstractFileOutputOperator<Object> +{ + private static final String NL = System.lineSeparator(); + private static final Charset CS = StandardCharsets.UTF_8; + + @NotNull + private String baseName; + + @Override + public byte[] getBytesForTuple(Object t) { + String result = new String(t.toString().getBytes(), CS) + NL; + return result.getBytes(CS); + } + + @Override + protected String getFileName(Object tuple) { + return baseName; + } + + public String getBaseName() { return baseName; } + public void setBaseName(String v) { baseName = v; } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJO.java b/examples/enricher/src/main/java/com/example/myapexapp/POJO.java new file mode 100644 index 0000000..32845e8 --- /dev/null +++ b/examples/enricher/src/main/java/com/example/myapexapp/POJO.java @@ -0,0 +1,49 @@ +package com.example.myapexapp; + +public class POJO +{ + private String phone; + private String imei; + private String imsi; + private int circleId; + + public String getPhone() + { + return phone; + } + + public void setPhone(String phone) + { + this.phone = phone; + } + + public String getImei() + { + return imei; + } + + public void setImei(String imei) + { + this.imei = imei; + } + + public String getImsi() + { + return imsi; + } + + public void setImsi(String imsi) + { + this.imsi = imsi; + } + + public int getCircleId() + { + return circleId; + } + + public void setCircleId(int circleId) + { + this.circleId = circleId; + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java b/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java new file mode 100644 index 0000000..bed2cfb --- /dev/null +++ b/examples/enricher/src/main/java/com/example/myapexapp/POJOEnriched.java @@ -0,0 +1,71 @@ +package com.example.myapexapp; + +public class POJOEnriched +{ + private String phone; + private String imei; + private String imsi; + private int circleId; + private String circleName; + + public String getPhone() + { + return phone; + } + + public void setPhone(String phone) + { + this.phone = phone; + } + + public String getImei() + { + return imei; + } + + public void setImei(String imei) + { + this.imei = imei; + } + + public String getImsi() + { + return imsi; + } + + public void setImsi(String imsi) + { + this.imsi = imsi; + } + + public int getCircleId() + { + return circleId; + } + + public void setCircleId(int circleId) + { + this.circleId = circleId; + } + + public String getCircleName() + { + return circleName; + } + + public void setCircleName(String circleName) + { + this.circleName = circleName; + } + + @Override public String toString() + { + return "POJOEnriched{" + + "phone='" + phone + '\'' + + ", imei='" + imei + '\'' + + ", imsi='" + imsi + '\'' + + ", circleId=" + circleId + + ", circleName='" + circleName + '\'' + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/resources/META-INF/properties.xml ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/resources/META-INF/properties.xml b/examples/enricher/src/main/resources/META-INF/properties.xml new file mode 100644 index 0000000..9ecf899 --- /dev/null +++ b/examples/enricher/src/main/resources/META-INF/properties.xml @@ -0,0 +1,41 @@ +<?xml version="1.0"?> +<configuration> + <!-- Parser --> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.Parser.port.out.attr.TUPLE_CLASS</name> + <value>com.example.myapexapp.POJO</value> + </property> + + <!-- Enrich --> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.input.attr.TUPLE_CLASS</name> + <value>com.example.myapexapp.POJO</value> + </property> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.port.output.attr.TUPLE_CLASS</name> + <value>com.example.myapexapp.POJOEnriched</value> + </property> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.Enrich.prop.store.fileName</name> + <value>/tmp/circleMapping.txt</value> + </property> + + <!-- file output operator --> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.filePath</name> + <value>/tmp/enrichedData</value> + </property> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.baseName</name> + <value>test</value> + </property> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.maxLength</name> + <value>1024</value> + </property> + <property> + <name>dt.application.EnricherAppWithJSONFile.operator.fileOut.prop.rotationWindows</name> + <value>4</value> + </property> +</configuration> + http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/main/resources/circleMapping.txt ---------------------------------------------------------------------- diff --git a/examples/enricher/src/main/resources/circleMapping.txt b/examples/enricher/src/main/resources/circleMapping.txt new file mode 100644 index 0000000..a9db91c --- /dev/null +++ b/examples/enricher/src/main/resources/circleMapping.txt @@ -0,0 +1,10 @@ +{"circleId":0, "circleName":"A"} +{"circleId":1, "circleName":"B"} +{"circleId":2, "circleName":"C"} +{"circleId":3, "circleName":"D"} +{"circleId":4, "circleName":"E"} +{"circleId":5, "circleName":"F"} +{"circleId":6, "circleName":"G"} +{"circleId":7, "circleName":"H"} +{"circleId":8, "circleName":"I"} +{"circleId":9, "circleName":"J"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java ---------------------------------------------------------------------- diff --git a/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java b/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java new file mode 100644 index 0000000..4b04603 --- /dev/null +++ b/examples/enricher/src/test/java/com/example/myapexapp/ApplicationTest.java @@ -0,0 +1,31 @@ +package com.example.myapexapp; + +import javax.validation.ConstraintViolationException; + +import org.junit.Assert; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +import com.datatorrent.api.LocalMode; + +/** + * Test the DAG declaration in local mode. + */ +public class ApplicationTest { + + @Test + public void testApplication() throws Exception { + try { + LocalMode lma = LocalMode.newInstance(); + Configuration conf = new Configuration(false); + conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml")); + lma.prepareDAG(new EnricherAppWithJSONFile(), conf); + LocalMode.Controller lc = lma.getController(); + lc.run(10000); // runs for 10 seconds and quits + } catch (ConstraintViolationException e) { + Assert.fail("constraint violations: " + e.getConstraintViolations()); + } + } + +} http://git-wip-us.apache.org/repos/asf/apex-malhar/blob/dd509f2a/examples/enricher/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/examples/enricher/src/test/resources/log4j.properties b/examples/enricher/src/test/resources/log4j.properties new file mode 100644 index 0000000..3bfcdc5 --- /dev/null +++ b/examples/enricher/src/test/resources/log4j.properties @@ -0,0 +1,21 @@ +log4j.rootLogger=DEBUG,CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n + +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} %M - %m%n +log4j.appender.RFA.File=/tmp/app.log + +# to enable, add SYSLOG to rootLogger +log4j.appender.SYSLOG=org.apache.log4j.net.SyslogAppender +log4j.appender.SYSLOG.syslogHost=127.0.0.1 +log4j.appender.SYSLOG.layout=org.apache.log4j.PatternLayout +log4j.appender.SYSLOG.layout.conversionPattern=${dt.cid} %-5p [%t] %c{2} %x - %m%n +log4j.appender.SYSLOG.Facility=LOCAL1 + +log4j.logger.org=info +#log4j.logger.org.apache.commons.beanutils=warn +log4j.logger.com.datatorrent=debug
