start of implementation for STREAMS-398
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/50f67bf8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/50f67bf8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/50f67bf8 Branch: refs/heads/invalid_headers Commit: 50f67bf8d97a61bb325f8524472f5c20e86287d0 Parents: 0e7fbcb Author: Steve Blackmon (@steveblackmon) <sblack...@apache.org> Authored: Tue Dec 29 17:19:15 2015 -0600 Committer: Steve Blackmon @steveblackmon <sblack...@apache.org> Committed: Wed Jun 1 12:49:27 2016 -0500 ---------------------------------------------------------------------- streams-plugins/pom.xml | 56 ++++ streams-plugins/streams-plugin-hive/pom.xml | 232 +++++++++++++++ .../apache/streams/plugins/StreamsPojoHive.java | 242 +++++++++++++++ .../streams/plugins/StreamsPojoHiveMojo.java | 71 +++++ .../plugins/test/StreamsPojoHiveTest.java | 55 ++++ .../src/test/resources/Tweet.hql | 297 +++++++++++++++++++ .../test/resources/streams-plugin-hive/pom.xml | 42 +++ 7 files changed, 995 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/pom.xml b/streams-plugins/pom.xml new file mode 100644 index 0000000..2306aab --- /dev/null +++ b/streams-plugins/pom.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>streams-project</artifactId> + <groupId>org.apache.streams</groupId> + <version>0.3-incubating-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>streams-plugins</artifactId> + + <packaging>pom</packaging> + <name>streams-plugins</name> + + <properties> + + </properties> + + <modules> + <module>streams-plugin-scala</module> + </modules> + + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-config</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-pojo</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </dependencyManagement> +</project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/pom.xml new file mode 100644 index 0000000..cbc055b --- /dev/null +++ b/streams-plugins/streams-plugin-hive/pom.xml @@ -0,0 +1,232 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-hive</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <packaging>maven-plugin</packaging> + + <parent> + <groupId>org.apache.streams</groupId> + <artifactId>streams-plugins</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <properties> + <hive.version>1.14</hive.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-config</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.jsonschema2pojo</groupId> + <artifactId>jsonschema2pojo-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.streams</groupId> + <artifactId>streams-pojo</artifactId> + <version>${project.version}</version> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>0.9.9</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-aether-provider</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-artifact</artifactId> + <version>3.3.3</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-compat</artifactId> + <version>3.3.3</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <version>3.3.3</version> + <exclusions> + <exclusion> + <groupId>org.sonatype.aether</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>3.3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.7</version> + <exclusions> + <exclusion> + <groupId>org.apache.maven</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + <version>3.0.15</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <exclusions> + <exclusion> + <artifactId>commons-logging</artifactId> + <groupId>commons-logging</groupId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-api</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-tools-generators</artifactId> + <version>3.3</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-testing</groupId> + <artifactId>maven-plugin-testing-harness</artifactId> + <version>3.3.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.6</version> + </dependency> + <dependency> + <groupId>joda-time</groupId> + <artifactId>joda-time</artifactId> + <version>2.2</version> + </dependency> + <dependency> + <groupId>org.joda</groupId> + <artifactId>joda-convert</artifactId> + <version>1.8.1</version> + </dependency> + </dependencies> + + <build> + <sourceDirectory>src/main/java</sourceDirectory> + <testSourceDirectory>src/test/java</testSourceDirectory> + <resources> + <resource> + <directory>src/main/resources</directory> + </resource> + </resources> + <testResources> + <testResource> + <directory>src/test/resources</directory> + </testResource> + </testResources> + <plugins> + <plugin> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.4</version> + <configuration> + <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound> + </configuration> + <executions> + <execution> + <id>mojo-descriptor</id> + <phase>process-classes</phase> + <goals> + <goal>descriptor</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.reflections</groupId> + <artifactId>reflections-maven</artifactId> + <executions> + <execution> + <goals> + <goal>reflections</goal> + </goals> + <phase>process-classes</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <id>streams-pojo-resource-dependencies</id> + <phase>process-test-resources</phase> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <configuration> + <includeArtifactIds>streams-pojo</includeArtifactIds> + <includes>org/apache/streams/pojo/json/**</includes> + <outputDirectory>${project.build.directory}/test-classes</outputDirectory> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java new file mode 100644 index 0000000..f8a00f7 --- /dev/null +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHive.java @@ -0,0 +1,242 @@ +package org.apache.streams.plugins; + +import com.google.common.base.Strings; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.streams.data.DocumentClassifier; +import org.reflections.ReflectionUtils; +import org.reflections.Reflections; +import org.reflections.scanners.SubTypesScanner; +import org.reflections.scanners.TypeAnnotationsScanner; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; +import org.reflections.ReflectionUtils.*; +import org.reflections.util.FilterBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Generated; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by sblackmon on 11/18/15. + */ +public class StreamsPojoHive implements Runnable { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHive.class); + + private final static String LS = System.getProperty("line.separator"); + + private StreamsPojoHiveMojo mojo; + + String outDir = "./target/generated-sources/hive"; + String packages[] = {"org.apache.streams.pojo.json"}; + + private final Reflections reflections = new Reflections( + new ConfigurationBuilder() + .forPackages(packages) + .filterInputsBy(new FilterBuilder().includePackage(packages)) + .setScanners( + new SubTypesScanner(), + new TypeAnnotationsScanner())); + + public void main(String[] args) { + StreamsPojoHive streamsPojoScala = new StreamsPojoHive(); + Thread thread = new Thread(streamsPojoScala); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + return; + } + + public StreamsPojoHive(StreamsPojoHiveMojo mojo) { + this.mojo = mojo; + if ( mojo != null && + mojo.getTarget() != null && + !Strings.isNullOrEmpty(mojo.getTarget().getAbsolutePath()) + ) + outDir = mojo.getTarget().getAbsolutePath(); + + if ( mojo != null && + mojo.getPackages() != null && + mojo.getPackages().length > 0 + ) + packages = mojo.getPackages(); + } + + public StreamsPojoHive() { + } + + public void run() { + + List<Class<?>> serializableClasses = detectSerializableClasses(); + + LOGGER.info("Detected {} serialiables:", serializableClasses.size()); + for( Class clazz : serializableClasses ) + LOGGER.debug(clazz.toString()); + + List<Class<?>> pojoClasses = detectPojoClasses(serializableClasses); + + LOGGER.info("Detected {} pojos:", pojoClasses.size()); + for( Class clazz : pojoClasses ) { + LOGGER.debug(clazz.toString()); + + } + + + for( Class clazz : pojoClasses ) { + String pojoPath = clazz.getPackage().getName().replace(".pojo.json", ".hive").replace(".","/")+"/"; + String pojoName = clazz.getSimpleName()+".hql"; + String pojoHive = renderPojo(clazz); + writeFile(outDir+"/"+pojoPath+pojoName, pojoHive); + } + + } + + private void writeFile(String pojoFile, String pojoHive) { + try { + File path = new File(pojoFile); + File dir = path.getParentFile(); + if( !dir.exists() ) + dir.mkdirs(); + Files.write(Paths.get(pojoFile), pojoHive.getBytes(), StandardOpenOption.CREATE_NEW); + } catch (Exception e) { + LOGGER.error("Write Exception: {}", e); + } + } + + public List<Class<?>> detectSerializableClasses() { + + Set<Class<? extends Serializable>> classes = + reflections.getSubTypesOf(java.io.Serializable.class); + + List<Class<?>> result = Lists.newArrayList(); + + for( Class clazz : classes ) { + result.add(clazz); + } + + return result; + } + + public List<Class<?>> detectPojoClasses(List<Class<?>> classes) { + + List<Class<?>> result = Lists.newArrayList(); + + for( Class clazz : classes ) { + try { + clazz.newInstance().toString(); + } catch( Exception e) {} + // super-halfass way to know if this is a jsonschema2pojo + if( clazz.getAnnotations().length >= 1 ) + result.add(clazz); + } + + return result; + } + + public String renderPojo(Class<?> pojoClass) { + StringBuffer stringBuffer = new StringBuffer(); + stringBuffer.append("CREATE TABLE "); + stringBuffer.append(pojoClass.getPackage().getName().replace(".pojo.json", ".hive")); + stringBuffer.append(LS); + stringBuffer.append("("); + stringBuffer.append(LS); + + Set<Field> fields = ReflectionUtils.getAllFields(pojoClass); + appendFields(stringBuffer, fields, "", ","); + + stringBuffer.append(")"); + + return stringBuffer.toString(); + } + + private void appendFields(StringBuffer stringBuffer, Set<Field> fields, String varDef, String fieldDelimiter) { + if( fields.size() > 0 ) { + stringBuffer.append(LS); + Map<String,Field> fieldsToAppend = uniqueFields(fields); + for( Iterator<Field> iter = fieldsToAppend.values().iterator(); iter.hasNext(); ) { + Field field = iter.next(); + stringBuffer.append(name(field)); + stringBuffer.append(": "); + stringBuffer.append(type(field)); + if( iter.hasNext()) stringBuffer.append(fieldDelimiter); + stringBuffer.append(LS); + } + } else { + stringBuffer.append(LS); + } + } + + private String value(Field field) { + if( field.getName().equals("verb")) { + return "\"post\""; + } else if( field.getName().equals("objectType")) { + return "\"application\""; + } else return null; + } + + private String type(Field field) { + if( field.getType().equals(java.lang.String.class)) { + return "STRING"; + } else if( field.getType().equals(java.lang.Integer.class)) { + return "INT"; + } else if( field.getType().equals(org.joda.time.DateTime.class)) { + return "DATE"; + }else if( field.getType().equals(java.util.Map.class)) { + return "MAP"; + } else if( field.getType().equals(java.util.List.class)) { + return "ARRAY"; + } + return field.getType().getCanonicalName().replace(".pojo.json", ".scala"); + } + + private Map<String,Field> uniqueFields(Set<Field> fieldset) { + Map<String,Field> fields = Maps.newTreeMap(); + Field item = null; + for( Iterator<Field> it = fieldset.iterator(); it.hasNext(); item = it.next() ) { + if( item != null && item.getName() != null ) { + Field added = fields.put(item.getName(), item); + } + // ensure right class will get used + } + return fields; + } + + private String name(Field field) { + if( field.getName().equals("object")) + return "obj"; + else return field.getName(); + } + + private boolean override(Field field) { + try { + if( field.getDeclaringClass().getSuperclass().getField(field.getName()) != null ) + return true; + else return false; + } catch( Exception e ) { + return false; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java new file mode 100644 index 0000000..a0af5ac --- /dev/null +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/StreamsPojoHiveMojo.java @@ -0,0 +1,71 @@ +package org.apache.streams.plugins; + +import org.apache.maven.artifact.repository.ArtifactRepository; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.descriptor.PluginDescriptor; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Execute; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.project.MavenProject; +import org.apache.maven.settings.Settings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +@Mojo( name = "hive", + defaultPhase = LifecyclePhase.GENERATE_SOURCES +) +@Execute( goal = "hive", + phase = LifecyclePhase.GENERATE_SOURCES +) +public class StreamsPojoHiveMojo extends AbstractMojo { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHiveMojo.class); + + @Component + private MavenProject project; + +// @Component +// private Settings settings; +// +// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) +// protected ArtifactRepository localRepository; +// +// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only +// private PluginDescriptor plugin; +// + @Parameter( defaultValue = "${project.basedir}", readonly = true ) + private File basedir; + + @Parameter(defaultValue = "${project.build.directory}", readonly = true) + private File target; + + @Parameter(defaultValue = "org.apache.streams.pojo.json", readonly = true) + private String[] packages; + + public void execute() throws MojoExecutionException { + StreamsPojoHive streamsPojoScala = new StreamsPojoHive(this); + Thread thread = new Thread(streamsPojoScala); + thread.start(); + try { + thread.join(); + } catch (InterruptedException e) { + LOGGER.error("InterruptedException", e); + } catch (Exception e) { + LOGGER.error("Exception", e); + } + return; + } + + public File getTarget() { + return target; + } + + public String[] getPackages() { + return packages; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java new file mode 100644 index 0000000..e5a7abd --- /dev/null +++ b/streams-plugins/streams-plugin-hive/src/test/java/org/apache/streams/plugins/test/StreamsPojoHiveTest.java @@ -0,0 +1,55 @@ +package org.apache.streams.plugins.test; + +import org.apache.streams.plugins.StreamsPojoHive; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileFilter; + +/** + * Test that Activity beans are compatible with the example activities in the spec. + */ +public class StreamsPojoHiveTest { + + private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPojoHiveTest.class); + + /** + * Tests that all example activities can be loaded into Activity beans + * + * @throws Exception + */ + @Test + public void testStreamsPojoHive() throws Exception { + StreamsPojoHive streamsPojoHive = new StreamsPojoHive(); + streamsPojoHive.main(new String[0]); + + File testOutput = new File( "./target/generated-sources/hive/org/apache/streams/hive"); + FileFilter hqlFilter = new FileFilter() { + @Override + public boolean accept(File pathname) { + if( pathname.getName().endsWith(".hql") ) + return true; + return false; + } + }; + + assert( testOutput != null ); + assert( testOutput.exists() == true ); + assert( testOutput.isDirectory() == true ); + assert( testOutput.listFiles(hqlFilter).length == 11 ); +// assert( new File(testOutput + "/traits").exists() == true ); +// assert( new File(testOutput + "/traits").isDirectory() == true ); +// assert( new File(testOutput + "/traits").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/traits").listFiles(scalaFilter).length == 4 ); +// assert( new File(testOutput + "/objectTypes").exists() == true ); +// assert( new File(testOutput + "/objectTypes").isDirectory() == true ); +// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/objectTypes").listFiles(scalaFilter).length == 43 ); +// assert( new File(testOutput + "/verbs").exists() == true ); +// assert( new File(testOutput + "/verbs").isDirectory() == true ); +// assert( new File(testOutput + "/verbs").listFiles(scalaFilter) != null ); +// assert( new File(testOutput + "/verbs").listFiles(scalaFilter).length == 89 ); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql b/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql new file mode 100644 index 0000000..dfd3a72 --- /dev/null +++ b/streams-plugins/streams-plugin-hive/src/test/resources/Tweet.hql @@ -0,0 +1,297 @@ +/* +root + |-- contributors: string (nullable = true) + |-- coordinates: string (nullable = true) + |-- created_at: string (nullable = true) + |-- entities: struct (nullable = true) + | |-- hashtags: array (nullable = true) + | | |-- element: struct (containsNull = true) + | | | |-- indices: array (nullable = true) + | | | | |-- element: long (containsNull = true) + | | | |-- text: string (nullable = true) + | |-- symbols: array (nullable = true) + | | |-- element: string (containsNull = true) + | |-- urls: array (nullable = true) + | | |-- element: struct (containsNull = true) + | | | |-- display_url: string (nullable = true) + | | | |-- expanded_url: string (nullable = true) + | | | |-- indices: array (nullable = true) + | | | | |-- element: long (containsNull = true) + | | | |-- url: string (nullable = true) + | |-- user_mentions: array (nullable = true) + | | |-- element: struct (containsNull = true) + | | | |-- id: long (nullable = true) + | | | |-- id_str: string (nullable = true) + | | | |-- indices: array (nullable = true) + | | | | |-- element: long (containsNull = true) + | | | |-- name: string (nullable = true) + | | | |-- screen_name: string (nullable = true) + |-- favorite_count: long (nullable = true) + |-- favorited: boolean (nullable = true) + |-- geo: string (nullable = true) + |-- id: long (nullable = true) + |-- id_str: string (nullable = true) + |-- in_reply_to_screen_name: string (nullable = true) + |-- in_reply_to_status_id: long (nullable = true) + |-- in_reply_to_status_id_str: string (nullable = true) + |-- in_reply_to_user_id: long (nullable = true) + |-- in_reply_to_user_id_str: string (nullable = true) + |-- lang: string (nullable = true) + |-- place: struct (nullable = true) + | |-- attributes: struct (nullable = true) + | |-- bounding_box: struct (nullable = true) + | | |-- coordinates: array (nullable = true) + | | | |-- element: array (containsNull = true) + | | | | |-- element: array (containsNull = true) + | | | | | |-- element: double (containsNull = true) + | | |-- type: string (nullable = true) + | |-- contained_within: array (nullable = true) + | | |-- element: string (containsNull = true) + | |-- country: string (nullable = true) + | |-- country_code: string (nullable = true) + | |-- full_name: string (nullable = true) + | |-- id: string (nullable = true) + | |-- name: string (nullable = true) + | |-- place_type: string (nullable = true) + | |-- url: string (nullable = true) + |-- possibly_sensitive: boolean (nullable = true) + |-- retweet_count: long (nullable = true) + |-- retweeted: boolean (nullable = true) + |-- retweeted_status: struct (nullable = true) + | |-- contributors: string (nullable = true) + | |-- coordinates: string (nullable = true) + | |-- created_at: string (nullable = true) + | |-- entities: struct (nullable = true) + | | |-- hashtags: array (nullable = true) + | | | |-- element: struct (containsNull = true) + | | | | |-- indices: array (nullable = true) + | | | | | |-- element: long (containsNull = true) + | | | | |-- text: string (nullable = true) + | | |-- symbols: array (nullable = true) + | | | |-- element: string (containsNull = true) + | | |-- urls: array (nullable = true) + | | | |-- element: struct (containsNull = true) + | | | | |-- display_url: string (nullable = true) + | | | | |-- expanded_url: string (nullable = true) + | | | | |-- indices: array (nullable = true) + | | | | | |-- element: long (containsNull = true) + | | | | |-- url: string (nullable = true) + | | |-- user_mentions: array (nullable = true) + | | | |-- element: struct (containsNull = true) + | | | | |-- id: long (nullable = true) + | | | | |-- id_str: string (nullable = true) + | | | | |-- indices: array (nullable = true) + | | | | | |-- element: long (containsNull = true) + | | | | |-- name: string (nullable = true) + | | | | |-- screen_name: string (nullable = true) + | |-- favorite_count: long (nullable = true) + | |-- favorited: boolean (nullable = true) + | |-- geo: string (nullable = true) + | |-- id: long (nullable = true) + | |-- id_str: string (nullable = true) + | |-- in_reply_to_screen_name: string (nullable = true) + | |-- in_reply_to_status_id: string (nullable = true) + | |-- in_reply_to_status_id_str: string (nullable = true) + | |-- in_reply_to_user_id: string (nullable = true) + | |-- in_reply_to_user_id_str: string (nullable = true) + | |-- lang: string (nullable = true) + | |-- place: struct (nullable = true) + | | |-- attributes: struct (nullable = true) + | | |-- bounding_box: struct (nullable = true) + | | | |-- coordinates: array (nullable = true) + | | | | |-- element: array (containsNull = true) + | | | | | |-- element: array (containsNull = true) + | | | | | | |-- element: double (containsNull = true) + | | | |-- type: string (nullable = true) + | | |-- contained_within: array (nullable = true) + | | | |-- element: string (containsNull = true) + | | |-- country: string (nullable = true) + | | |-- country_code: string (nullable = true) + | | |-- full_name: string (nullable = true) + | | |-- id: string (nullable = true) + | | |-- name: string (nullable = true) + | | |-- place_type: string (nullable = true) + | | |-- url: string (nullable = true) + | |-- possibly_sensitive: boolean (nullable = true) + | |-- retweet_count: long (nullable = true) + | |-- retweeted: boolean (nullable = true) + | |-- source: string (nullable = true) + | |-- text: string (nullable = true) + | |-- truncated: boolean (nullable = true) + | |-- user: struct (nullable = true) + | | |-- contributors_enabled: boolean (nullable = true) + | | |-- created_at: string (nullable = true) + | | |-- default_profile: boolean (nullable = true) + | | |-- default_profile_image: boolean (nullable = true) + | | |-- description: string (nullable = true) + | | |-- entities: struct (nullable = true) + | | | |-- description: struct (nullable = true) + | | | | |-- urls: array (nullable = true) + | | | | | |-- element: struct (containsNull = true) + | | | | | | |-- display_url: string (nullable = true) + | | | | | | |-- expanded_url: string (nullable = true) + | | | | | | |-- indices: array (nullable = true) + | | | | | | | |-- element: long (containsNull = true) + | | | | | | |-- url: string (nullable = true) + | | | |-- url: struct (nullable = true) + | | | | |-- urls: array (nullable = true) + | | | | | |-- element: struct (containsNull = true) + | | | | | | |-- display_url: string (nullable = true) + | | | | | | |-- expanded_url: string (nullable = true) + | | | | | | |-- indices: array (nullable = true) + | | | | | | | |-- element: long (containsNull = true) + | | | | | | |-- url: string (nullable = true) + | | |-- favourites_count: long (nullable = true) + | | |-- follow_request_sent: boolean (nullable = true) + | | |-- followers_count: long (nullable = true) + | | |-- following: boolean (nullable = true) + | | |-- friends_count: long (nullable = true) + | | |-- geo_enabled: boolean (nullable = true) + | | |-- id: long (nullable = true) + | | |-- id_str: string (nullable = true) + | | |-- is_translation_enabled: boolean (nullable = true) + | | |-- is_translator: boolean (nullable = true) + | | |-- lang: string (nullable = true) + | | |-- listed_count: long (nullable = true) + | | |-- location: string (nullable = true) + | | |-- name: string (nullable = true) + | | |-- notifications: boolean (nullable = true) + | | |-- profile_background_color: string (nullable = true) + | | |-- profile_background_image_url: string (nullable = true) + | | |-- profile_background_image_url_https: string (nullable = true) + | | |-- profile_background_tile: boolean (nullable = true) + | | |-- profile_banner_url: string (nullable = true) + | | |-- profile_image_url: string (nullable = true) + | | |-- profile_image_url_https: string (nullable = true) + | | |-- profile_link_color: string (nullable = true) + | | |-- profile_sidebar_border_color: string (nullable = true) + | | |-- profile_sidebar_fill_color: string (nullable = true) + | | |-- profile_text_color: string (nullable = true) + | | |-- profile_use_background_image: boolean (nullable = true) + | | |-- protected: boolean (nullable = true) + | | |-- screen_name: string (nullable = true) + | | |-- statuses_count: long (nullable = true) + | | |-- time_zone: string (nullable = true) + | | |-- url: string (nullable = true) + | | |-- utc_offset: long (nullable = true) + | | |-- verified: boolean (nullable = true) + |-- source: string (nullable = true) + |-- text: string (nullable = true) + |-- truncated: boolean (nullable = true) + |-- user: struct (nullable = true) + | |-- contributors_enabled: boolean (nullable = true) + | |-- created_at: string (nullable = true) + | |-- default_profile: boolean (nullable = true) + | |-- default_profile_image: boolean (nullable = true) + | |-- description: string (nullable = true) + | |-- entities: struct (nullable = true) + | | |-- description: struct (nullable = true) + | | | |-- urls: array (nullable = true) + | | | | |-- element: string (containsNull = true) + | | |-- url: struct (nullable = true) + | | | |-- urls: array (nullable = true) + | | | | |-- element: struct (containsNull = true) + | | | | | |-- display_url: string (nullable = true) + | | | | | |-- expanded_url: string (nullable = true) + | | | | | |-- indices: array (nullable = true) + | | | | | | |-- element: long (containsNull = true) + | | | | | |-- url: string (nullable = true) + | |-- favourites_count: long (nullable = true) + | |-- follow_request_sent: boolean (nullable = true) + | |-- followers_count: long (nullable = true) + | |-- following: boolean (nullable = true) + | |-- friends_count: long (nullable = true) + | |-- geo_enabled: boolean (nullable = true) + | |-- id: long (nullable = true) + | |-- id_str: string (nullable = true) + | |-- is_translation_enabled: boolean (nullable = true) + | |-- is_translator: boolean (nullable = true) + | |-- lang: string (nullable = true) + | |-- listed_count: long (nullable = true) + | |-- location: string (nullable = true) + | |-- name: string (nullable = true) + | |-- notifications: boolean (nullable = true) + | |-- profile_background_color: string (nullable = true) + | |-- profile_background_image_url: string (nullable = true) + | |-- profile_background_image_url_https: string (nullable = true) + | |-- profile_background_tile: boolean (nullable = true) + | |-- profile_banner_url: string (nullable = true) + | |-- profile_image_url: string (nullable = true) + | |-- profile_image_url_https: string (nullable = true) + | |-- profile_link_color: string (nullable = true) + | |-- profile_sidebar_border_color: string (nullable = true) + | |-- profile_sidebar_fill_color: string (nullable = true) + | |-- profile_text_color: string (nullable = true) + | |-- profile_use_background_image: boolean (nullable = true) + | |-- protected: boolean (nullable = true) + | |-- screen_name: string (nullable = true) + | |-- statuses_count: long (nullable = true) + | |-- time_zone: string (nullable = true) + | |-- url: string (nullable = true) + | |-- utc_offset: long (nullable = true) + | |-- verified: boolean (nullable = true) +*/ +create table tweet ( + created_at string, + entities struct + < + hashtags: array + < + struct + < + text: string + > + >, + urls: array + < + struct + < + expanded_url: string + > + >, + user_mentions: array + < + struct + < + id_str: string + > + > + >, + in_reply_to_user_id_str: string, + user: struct + < + id_str: string + >, + retweeted_status struct + < + entities struct + < + hashtags: array + < + struct + < + text: string + > + >, + urls: array + < + struct + < + expanded_url: string + > + >, + user_mentions: array + < + struct + < + id_str: string + > + > + >, + in_reply_to_user_id_str: string, + user: struct + < + id_str: string + > + > +) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/50f67bf8/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml new file mode 100644 index 0000000..b1de7b8 --- /dev/null +++ b/streams-plugins/streams-plugin-hive/src/test/resources/streams-plugin-hive/pom.xml @@ -0,0 +1,42 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-hive-test</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <packaging>jar</packaging> + <name>Test StreamsPojoHiveMojo</name> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.streams.plugins</groupId> + <artifactId>streams-plugin-hive</artifactId> + <version>0.3-incubating-SNAPSHOT</version> + <configuration> + <packages> + <package>org.apache.streams.pojo.json</package> + </packages> + <target>target/test-classes/streams-hive-plugin/</target> + </configuration> + <executions> + <execution> + <goals> + <goal>hive</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file
