http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java index 1485023..ea512b3 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/main/java/org/apache/streams/plugins/elasticsearch/StreamsElasticsearchResourceGeneratorMojo.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.streams.plugins.elasticsearch; import org.apache.maven.plugin.AbstractMojo; @@ -33,58 +34,59 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "generate-resources", - defaultPhase = LifecyclePhase.GENERATE_RESOURCES -) -@Execute( goal = "generate-resources", - phase = LifecyclePhase.GENERATE_RESOURCES -) +@Mojo ( name = "generate-resources", + defaultPhase = LifecyclePhase.GENERATE_RESOURCES + ) +@Execute ( goal = "generate-resources", + phase = LifecyclePhase.GENERATE_RESOURCES + ) public class StreamsElasticsearchResourceGeneratorMojo extends AbstractMojo { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojo.class); - private volatile MojoFailureException mojoFailureException; + private volatile MojoFailureException mojoFailureException; - @Component - private MavenProject project; + @Component + private MavenProject project; -// @Component -// private Settings settings; -// -// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) -// protected ArtifactRepository localRepository; -// -// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only -// private PluginDescriptor plugin; -// - @Parameter( defaultValue = "${project.basedir}", readonly = true ) - private File basedir; + // @Component + // private Settings settings; + // + // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) + // protected ArtifactRepository localRepository; + // + // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only + // private PluginDescriptor plugin; + // + @Parameter( defaultValue = "${project.basedir}", readonly = true ) + private File basedir; - @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only - public String sourceDirectory; + @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only + public String sourceDirectory; - @Parameter( readonly = true ) // Maven 3 only - public List<String> sourcePaths; + @Parameter( readonly = true ) // Maven 3 only + public List<String> sourcePaths; - @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true) - public String targetDirectory; + @Parameter(defaultValue = "./target/generated-resources/streams-plugin-elasticsearch", readonly = true) + public String targetDirectory; - public void execute() throws MojoExecutionException, MojoFailureException { + public void execute() throws MojoExecutionException, MojoFailureException { - //addProjectDependenciesToClasspath(); + //addProjectDependenciesToClasspath(); - StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); - if( sourcePaths != null && sourcePaths.size() > 0) - config.setSourcePaths(sourcePaths); - else - config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory(targetDirectory); + if ( sourcePaths != null && sourcePaths.size() > 0) { + config.setSourcePaths(sourcePaths); + } else { + config.setSourceDirectory(sourceDirectory); + } + config.setTargetDirectory(targetDirectory); - StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); - streamsElasticsearchResourceGenerator.run(); + streamsElasticsearchResourceGenerator.run(); - } + } } \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java index 809b9a0..887461c 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorCLITest.java @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.streams.plugins.elasticsearch.test; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; + import com.google.common.collect.Lists; import com.google.common.io.Files; -import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; import org.junit.Test; import java.io.File; @@ -30,28 +32,28 @@ import java.util.List; import static org.apache.streams.plugins.elasticsearch.test.StreamsElasticsearchResourceGeneratorTest.jsonFilter; /** - * Created by sblackmon on 5/5/16. + * Test that StreamsElasticsearchResourceGeneratorCLI generates resources. */ public class StreamsElasticsearchResourceGeneratorCLITest { - @Test - public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception { + @Test + public void testStreamsElasticsearchResourceGeneratorCLI() throws Exception { - String sourceDirectory = "target/test-classes/activitystreams-schemas"; - String targetDirectory = "target/generated-resources/elasticsearch-cli"; + String sourceDirectory = "target/test-classes/activitystreams-schemas"; + String targetDirectory = "target/generated-resources/elasticsearch-cli"; - List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); - StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0])); + List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); + StreamsElasticsearchResourceGenerator.main(argsList.toArray(new String[0])); - File testOutput = new File(targetDirectory); + File testOutput = new File(targetDirectory); - assert( testOutput != null ); - assert( testOutput.exists() == true ); - assert( testOutput.isDirectory() == true ); + assert ( testOutput != null ); + assert ( testOutput.exists() == true ); + assert ( testOutput.isDirectory() == true ); - Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) - .filter(jsonFilter); - Collection<File> outputCollection = Lists.newArrayList(outputIterator); - assert( outputCollection.size() == 133 ); - } + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(jsonFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert ( outputCollection.size() == 133 ); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java index 2a24846..a0fbb99 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorMojoIT.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.streams.plugins.elasticsearch.test; import com.google.common.collect.Lists; @@ -31,38 +32,37 @@ import java.util.ArrayList; import java.util.List; /** - * Tests that streams-plugin-hive running via maven generates hql resources + * Tests that streams-plugin-elasticsearch running via maven generates elasticsearch mapping resources. */ public class StreamsElasticsearchResourceGeneratorMojoIT extends TestCase { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoIT.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorMojoIT.class); - protected void setUp() throws Exception - { - // required for mojo lookups to work - super.setUp(); - } + protected void setUp() throws Exception { + // required for mojo lookups to work + super.setUp(); + } - @Test - public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception { + @Test + public void testStreamsElasticsearchResourceGeneratorMojo() throws Exception { - File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" ); + File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-elasticsearch" ); - Verifier verifier; + Verifier verifier; - verifier = new Verifier( testDir.getAbsolutePath() ); + verifier = new Verifier( testDir.getAbsolutePath() ); - List cliOptions = new ArrayList(); - cliOptions.add( "-N" ); - verifier.executeGoals( Lists.<String>newArrayList( - "clean", - "dependency:unpack-dependencies", - "generate-resources")); + List cliOptions = new ArrayList(); + cliOptions.add( "-N" ); + verifier.executeGoals( Lists.<String>newArrayList( + "clean", + "dependency:unpack-dependencies", + "generate-resources")); - verifier.verifyErrorFreeLog(); + verifier.verifyErrorFreeLog(); - verifier.resetStreams(); + verifier.resetStreams(); - } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java index 9c1eed4..4322b11 100644 --- a/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-elasticsearch/src/test/java/org/apache/streams/plugins/elasticsearch/test/StreamsElasticsearchResourceGeneratorTest.java @@ -16,24 +16,26 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.streams.plugins.elasticsearch.test; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig; +import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; + import com.google.common.base.Predicate; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; -import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchGenerationConfig; -import org.apache.streams.plugins.elasticsearch.StreamsElasticsearchResourceGenerator; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.File; import java.util.Collection; import java.util.Iterator; +import javax.annotation.Nullable; import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; @@ -42,87 +44,91 @@ import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; */ public class StreamsElasticsearchResourceGeneratorTest { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class); - - public static final Predicate<File> jsonFilter = new Predicate<File>() { - @Override - public boolean apply(@Nullable File file) { - if( file.getName().endsWith(".json") ) - return true; - else return false; - } - }; + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsElasticsearchResourceGeneratorTest.class); - /** - * Test that Elasticsearch resources are generated - * - * @throws Exception - */ - @Test - public void StreamsElasticsearchResourceGenerator() throws Exception { - - StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); - - String sourceDirectory = "target/test-classes/activitystreams-schemas"; - - config.setSourceDirectory(sourceDirectory); - - config.setTargetDirectory("target/generated-resources/elasticsearch"); - - config.setExclusions(Sets.newHashSet("attachments")); + public static final Predicate<File> jsonFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if ( file.getName().endsWith(".json") ) { + return true; + } else { + return false; + } + } + }; - config.setMaxDepth(2); + /** + * Test that Elasticsearch resources are generated. + * + * @throws Exception Exception + */ + @Test + public void StreamsElasticsearchResourceGenerator() throws Exception { - StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); - streamsElasticsearchResourceGenerator.run(); + StreamsElasticsearchGenerationConfig config = new StreamsElasticsearchGenerationConfig(); - File testOutput = config.getTargetDirectory(); + String sourceDirectory = "target/test-classes/activitystreams-schemas"; - Predicate<File> jsonFilter = new Predicate<File>() { - @Override - public boolean apply(@Nullable File file) { - if( file.getName().endsWith(".json") ) - return true; - else return false; - } - }; + config.setSourceDirectory(sourceDirectory); - assert( testOutput != null ); - assert( testOutput.exists() == true ); - assert( testOutput.isDirectory() == true ); + config.setTargetDirectory("target/generated-resources/elasticsearch"); - Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) - .filter(jsonFilter); - Collection<File> outputCollection = Lists.newArrayList(outputIterator); - assert( outputCollection.size() == 133 ); + config.setExclusions(Sets.newHashSet("attachments")); - String expectedDirectory = "target/test-classes/expected"; - File testExpected = new File( expectedDirectory ); + config.setMaxDepth(2); - Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) - .filter(jsonFilter); - Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); + StreamsElasticsearchResourceGenerator streamsElasticsearchResourceGenerator = new StreamsElasticsearchResourceGenerator(config); + streamsElasticsearchResourceGenerator.run(); - int fails = 0; + File testOutput = config.getTargetDirectory(); - Iterator<File> iterator = expectedCollection.iterator(); - while( iterator.hasNext() ) { - File objectExpected = iterator.next(); - String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); - File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); - LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); - assert( objectActual.exists()); - if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { - LOGGER.info("Exact Match!"); - } else { - LOGGER.info("No Match!"); - fails++; - } - } - if( fails > 0 ) { - LOGGER.info("Fails: {}", fails); - Assert.fail(); + Predicate<File> jsonFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if ( file.getName().endsWith(".json") ) { + return true; + } else { + return false; } + } + }; + assert ( testOutput != null ); + assert ( testOutput.exists() == true ); + assert ( testOutput.isDirectory() == true ); + + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(jsonFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert ( outputCollection.size() == 133 ); + + String expectedDirectory = "target/test-classes/expected"; + File testExpected = new File( expectedDirectory ); + + Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) + .filter(jsonFilter); + Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); + + int fails = 0; + + Iterator<File> iterator = expectedCollection.iterator(); + while ( iterator.hasNext() ) { + File objectExpected = iterator.next(); + String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); + File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); + LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); + assert ( objectActual.exists()); + if ( FileUtils.contentEquals(objectActual, objectExpected) == true ) { + LOGGER.info("Exact Match!"); + } else { + LOGGER.info("No Match!"); + fails++; + } } + if ( fails > 0 ) { + LOGGER.info("Fails: {}", fails); + Assert.fail(); + } + + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java index 977c3d9..d939372 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseGenerationConfig.java @@ -20,6 +20,7 @@ package org.apache.streams.plugins.hbase; import org.apache.streams.util.schema.GenerationConfig; + import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; @@ -33,77 +34,80 @@ import java.util.List; import java.util.Set; /** - * Configures StreamsHiveResourceGenerator - * - * + * Configures StreamsHiveResourceGenerator. */ public class StreamsHbaseGenerationConfig extends DefaultGenerationConfig implements GenerationConfig { - public String getSourceDirectory() { - return sourceDirectory; + public String getSourceDirectory() { + return sourceDirectory; + } + + public List<String> getSourcePaths() { + return sourcePaths; + } + + private String columnFamily; + private String sourceDirectory; + private List<String> sourcePaths = new ArrayList<String>(); + private String targetDirectory; + private int maxDepth = 1; + + public Set<String> getExclusions() { + return exclusions; + } + + public void setExclusions(Set<String> exclusions) { + this.exclusions = exclusions; + } + + private Set<String> exclusions = new HashSet<String>(); + + public int getMaxDepth() { + return maxDepth; + } + + public void setSourceDirectory(String sourceDirectory) { + this.sourceDirectory = sourceDirectory; + } + + public void setSourcePaths(List<String> sourcePaths) { + this.sourcePaths = sourcePaths; + } + + public void setTargetDirectory(String targetDirectory) { + this.targetDirectory = targetDirectory; + } + + public File getTargetDirectory() { + return new File(targetDirectory); + } + + /** + * get all sources. + * @return Iterator of URL + */ + public Iterator<URL> getSource() { + if (null != sourceDirectory) { + return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); } - - public List<String> getSourcePaths() { - return sourcePaths; + List<URL> sourceUrls = new ArrayList<URL>(); + if ( sourcePaths != null && sourcePaths.size() > 0) { + for (String source : sourcePaths) { + sourceUrls.add(URLUtil.parseURL(source)); + } } + return sourceUrls.iterator(); + } - private String columnFamily; - private String sourceDirectory; - private List<String> sourcePaths = new ArrayList<String>(); - private String targetDirectory; - private int maxDepth = 1; + public void setMaxDepth(int maxDepth) { + this.maxDepth = maxDepth; + } - public Set<String> getExclusions() { - return exclusions; - } - - public void setExclusions(Set<String> exclusions) { - this.exclusions = exclusions; - } + public String getColumnFamily() { + return columnFamily; + } - private Set<String> exclusions = new HashSet<String>(); - - public int getMaxDepth() { - return maxDepth; - } - - public void setSourceDirectory(String sourceDirectory) { - this.sourceDirectory = sourceDirectory; - } - - public void setSourcePaths(List<String> sourcePaths) { - this.sourcePaths = sourcePaths; - } - - public void setTargetDirectory(String targetDirectory) { - this.targetDirectory = targetDirectory; - } - - public File getTargetDirectory() { - return new File(targetDirectory); - } - - public Iterator<URL> getSource() { - if (null != sourceDirectory) { - return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); - } - List<URL> sourceURLs = new ArrayList<URL>(); - if( sourcePaths != null && sourcePaths.size() > 0) - for (String source : sourcePaths) { - sourceURLs.add(URLUtil.parseURL(source)); - } - return sourceURLs.iterator(); - } - - public void setMaxDepth(int maxDepth) { - this.maxDepth = maxDepth; - } - - public String getColumnFamily() { - return columnFamily; - } - - public void setColumnFamily(String columnFamily) { - this.columnFamily = columnFamily; - } + public void setColumnFamily(String columnFamily) { + this.columnFamily = columnFamily; + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java index 9f96fb8..cbea67f 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGenerator.java @@ -19,16 +19,17 @@ package org.apache.streams.plugins.hbase; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; import org.apache.streams.util.schema.FieldType; import org.apache.streams.util.schema.FieldUtil; import org.apache.streams.util.schema.GenerationConfig; import org.apache.streams.util.schema.Schema; import org.apache.streams.util.schema.SchemaStore; import org.apache.streams.util.schema.SchemaStoreImpl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,161 +53,188 @@ import static org.apache.streams.util.schema.FileUtil.writeFile; */ public class StreamsHbaseResourceGenerator implements Runnable { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class); - - private final static String LS = System.getProperty("line.separator"); - - private StreamsHbaseGenerationConfig config; - - private SchemaStore schemaStore = new SchemaStoreImpl(); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGenerator.class); - private int currentDepth = 0; + private static final String LS = System.getProperty("line.separator"); - public static void main(String[] args) { - StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + private StreamsHbaseGenerationConfig config; - String sourceDirectory = "src/main/jsonschema"; - String targetDirectory = "target/generated-resources/hbase"; + private SchemaStore schemaStore = new SchemaStoreImpl(); - if( args.length > 0 ) - sourceDirectory = args[0]; - if( args.length > 1 ) - targetDirectory = args[1]; + private int currentDepth = 0; - config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory(targetDirectory); + /** + * Run from CLI without Maven + * + * <p/> + * java -jar streams-plugin-hbase-jar-with-dependencies.jar StreamsHbaseResourceGenerator src/main/jsonschema target/generated-resources + * + * @param args [sourceDirectory, targetDirectory] + * */ + public static void main(String[] args) { + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); - StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); - streamsHbaseResourceGenerator.run(); + String sourceDirectory = "src/main/jsonschema"; + String targetDirectory = "target/generated-resources/hbase"; + if ( args.length > 0 ) { + sourceDirectory = args[0]; } - - public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) { - this.config = config; + if ( args.length > 1 ) { + targetDirectory = args[1]; } - public void run() { - - checkNotNull(config); + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); - generate(config); + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + streamsHbaseResourceGenerator.run(); - } + } - public void generate(StreamsHbaseGenerationConfig config) { + public StreamsHbaseResourceGenerator(StreamsHbaseGenerationConfig config) { + this.config = config; + } - LinkedList<File> sourceFiles = new LinkedList<File>(); + @Override + public void run() { - for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { - URL source = sources.next(); - sourceFiles.add(URLUtil.getFileFromURL(source)); - } + checkNotNull(config); - LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); + generate(config); - resolveRecursive((GenerationConfig)config, sourceFiles); + } - LOGGER.info("Resolved {} schema files:", sourceFiles.size()); + /** + * run generate using supplied StreamsHbaseGenerationConfig. + * @param config StreamsHbaseGenerationConfig + */ + public void generate(StreamsHbaseGenerationConfig config) { - for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { - File item = iterator.next(); - schemaStore.create(item.toURI()); - } + LinkedList<File> sourceFiles = new LinkedList<File>(); - LOGGER.info("Identified {} objects:", schemaStore.getSize()); + for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { + URL source = sources.next(); + sourceFiles.add(URLUtil.getFileFromURL(source)); + } - for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { - Schema schema = schemaIterator.next(); - currentDepth = 0; - if( schema.getURI().getScheme().equals("file")) { - String inputFile = schema.getURI().getPath(); - String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); - for (String sourcePath : config.getSourcePaths()) { - resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); - } - String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt"); + LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); - LOGGER.info("Processing {}:", resourcePath); + resolveRecursive((GenerationConfig)config, sourceFiles); - String resourceId = dropExtension(resourcePath).replace("/", "_"); + LOGGER.info("Resolved {} schema files:", sourceFiles.size()); - String resourceContent = generateResource(schema, resourceId); + for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { + File item = iterator.next(); + schemaStore.create(item.toURI()); + } - writeFile(outputFile, resourceContent); + LOGGER.info("Identified {} objects:", schemaStore.getSize()); - LOGGER.info("Wrote {}:", outputFile); - } + for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { + Schema schema = schemaIterator.next(); + currentDepth = 0; + if ( schema.getUri().getScheme().equals("file")) { + String inputFile = schema.getUri().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); } + String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "txt"); - } + LOGGER.info("Processing {}:", resourcePath); - public String generateResource(Schema schema, String resourceId) { - StringBuilder resourceBuilder = new StringBuilder(); - resourceBuilder.append("CREATE "); - resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId); - return resourceBuilder.toString(); - } + String resourceId = dropExtension(resourcePath).replace("/", "_"); - public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) { - checkNotNull(builder); - ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); - if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { - - List<String> fieldStrings = Lists.newArrayList(); - - // table - fieldStrings.add(hbaseEscape(schemaSymbol(schema))); - - // column family - fieldStrings.add(hbaseEscape(schemaSymbol(schema))); - - // parent column family - if( schema.getParent() != null ) - fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent()))); - - // sub-object column families - if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) { - - Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); - Joiner joiner = Joiner.on(", ").skipNulls(); - for( ; fields.hasNext(); ) { - Map.Entry<String, JsonNode> field = fields.next(); - String fieldId = field.getKey(); - if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { - ObjectNode fieldNode = (ObjectNode) field.getValue(); - FieldType fieldType = FieldUtil.determineFieldType(fieldNode); - if (fieldType != null ) { - switch (fieldType) { - case OBJECT: - fieldStrings.add(hbaseEscape(fieldId)); - } - } - } - } - builder.append(joiner.join(fieldStrings)); + String resourceContent = generateResource(schema, resourceId); - } - } - checkNotNull(builder); - return builder; - } + writeFile(outputFile, resourceContent); - private static String hbaseEscape( String fieldId ) { - return "'"+fieldId+"'"; + LOGGER.info("Wrote {}:", outputFile); + } } - private String schemaSymbol( Schema schema ) { - if (schema == null) return null; - if (schema.getURI().getScheme().equals("file")) { - String inputFile = schema.getURI().getPath(); - String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); - for (String sourcePath : config.getSourcePaths()) { - resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + + /** + * generateResource String from schema and resourceId. + * @param schema Schema + * @param resourceId String + * @return mapping + */ + public String generateResource(Schema schema, String resourceId) { + StringBuilder resourceBuilder = new StringBuilder(); + resourceBuilder.append("CREATE "); + resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId); + return resourceBuilder.toString(); + } + + protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId) { + checkNotNull(builder); + ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); + if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { + + List<String> fieldStrings = Lists.newArrayList(); + + // table + fieldStrings.add(hbaseEscape(schemaSymbol(schema))); + + // column family + fieldStrings.add(hbaseEscape(schemaSymbol(schema))); + + // parent column family + if ( schema.getParent() != null ) { + fieldStrings.add(hbaseEscape(schemaSymbol(schema.getParent()))); + } + + // sub-object column families + if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) { + + Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); + Joiner joiner = Joiner.on(", ").skipNulls(); + for ( ; fields.hasNext(); ) { + Map.Entry<String, JsonNode> field = fields.next(); + String fieldId = field.getKey(); + if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { + ObjectNode fieldNode = (ObjectNode) field.getValue(); + FieldType fieldType = FieldUtil.determineFieldType(fieldNode); + if (fieldType != null ) { + switch (fieldType) { + case OBJECT: + fieldStrings.add(hbaseEscape(fieldId)); + break; + default: + break; + } } - return dropExtension(resourcePath).replace("/", "_"); - } else { - return "IDK"; + } } + builder.append(joiner.join(fieldStrings)); + + } + } + checkNotNull(builder); + return builder; + } + + private static String hbaseEscape( String fieldId ) { + return "'" + fieldId + "'"; + } + + private String schemaSymbol( Schema schema ) { + if (schema == null) { + return null; + } + if (schema.getUri().getScheme().equals("file")) { + String inputFile = schema.getUri().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + return dropExtension(resourcePath).replace("/", "_"); + } else { + return "IDK"; } + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java index 24a6b88..addc657 100644 --- a/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java +++ b/streams-plugins/streams-plugin-hbase/src/main/java/org/apache/streams/plugins/hbase/StreamsHbaseResourceGeneratorMojo.java @@ -34,57 +34,65 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.util.List; -@Mojo( name = "generate-resources", - defaultPhase = LifecyclePhase.GENERATE_RESOURCES -) -@Execute( goal = "generate-resources", - phase = LifecyclePhase.GENERATE_RESOURCES -) +@Mojo ( + name = "generate-resources", + defaultPhase = LifecyclePhase.GENERATE_RESOURCES + ) +@Execute ( + goal = "generate-resources", + phase = LifecyclePhase.GENERATE_RESOURCES + ) public class StreamsHbaseResourceGeneratorMojo extends AbstractMojo { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojo.class); - private volatile MojoFailureException mojoFailureException; + private volatile MojoFailureException mojoFailureException; - @Component - private MavenProject project; + @Component + private MavenProject project; -// @Component -// private Settings settings; -// -// @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) -// protected ArtifactRepository localRepository; -// -// @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only -// private PluginDescriptor plugin; -// - @Parameter( defaultValue = "${project.basedir}", readonly = true ) - private File basedir; + // @Component + // private Settings settings; + // + // @Parameter( defaultValue = "${localRepository}", readonly = true, required = true ) + // protected ArtifactRepository localRepository; + // + // @Parameter( defaultValue = "${plugin}", readonly = true ) // Maven 3 only + // private PluginDescriptor plugin; + // + @Parameter( defaultValue = "${project.basedir}", readonly = true ) + private File basedir; - @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only - public String sourceDirectory; + @Parameter( defaultValue = "src/main/jsonschema", readonly = true ) // Maven 3 only + public String sourceDirectory; - @Parameter( readonly = true ) // Maven 3 only - public List<String> sourcePaths; + @Parameter( readonly = true ) // Maven 3 only + public List<String> sourcePaths; - @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true) - public String targetDirectory; + @Parameter(defaultValue = "./target/generated-resources/hbase", readonly = true) + public String targetDirectory; - public void execute() throws MojoExecutionException, MojoFailureException { + /** + * execute StreamsHbaseResourceGenerator mojo. + * @throws MojoExecutionException MojoExecutionException + * @throws MojoFailureException MojoFailureException + */ + public void execute() throws MojoExecutionException, MojoFailureException { - //addProjectDependenciesToClasspath(); + //addProjectDependenciesToClasspath(); - StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); - if( sourcePaths != null && sourcePaths.size() > 0) - config.setSourcePaths(sourcePaths); - else - config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory(targetDirectory); + if ( sourcePaths != null && sourcePaths.size() > 0) { + config.setSourcePaths(sourcePaths); + } else { + config.setSourceDirectory(sourceDirectory); + } + config.setTargetDirectory(targetDirectory); - StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); - streamsHbaseResourceGenerator.run(); + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + streamsHbaseResourceGenerator.run(); - } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java index 3c223eb..254bd0e 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorCLITest.java @@ -19,13 +19,12 @@ package org.apache.streams.plugins.test; -import com.google.common.base.Predicate; +import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; + import com.google.common.collect.Lists; import com.google.common.io.Files; -import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; import org.junit.Test; -import javax.annotation.Nullable; import java.io.File; import java.util.Collection; import java.util.List; @@ -33,28 +32,28 @@ import java.util.List; import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter; /** - * Created by sblackmon on 5/5/16. + * Test that StreamsHbaseResourceGeneratorCLI generates resources. */ public class StreamsHbaseResourceGeneratorCLITest { - @Test - public void testStreamsHiveResourceGeneratorCLI() throws Exception { + @Test + public void testStreamsHbaseResourceGeneratorCLI() throws Exception { - String sourceDirectory = "target/test-classes/activitystreams-schemas"; - String targetDirectory = "target/generated-resources/hbase-cli"; + String sourceDirectory = "target/test-classes/activitystreams-schemas"; + String targetDirectory = "target/generated-resources/hbase-cli"; - List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); - StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0])); + List<String> argsList = Lists.newArrayList(sourceDirectory, targetDirectory); + StreamsHbaseResourceGenerator.main(argsList.toArray(new String[0])); - File testOutput = new File(targetDirectory); + File testOutput = new File(targetDirectory); - assert( testOutput != null ); - assert( testOutput.exists() == true ); - assert( testOutput.isDirectory() == true ); + assert ( testOutput != null ); + assert ( testOutput.exists() == true ); + assert ( testOutput.isDirectory() == true ); - Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) - .filter(txtFilter); - Collection<File> outputCollection = Lists.newArrayList(outputIterator); - assert( outputCollection.size() == 133 ); - } + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert ( outputCollection.size() == 133 ); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java index 1495bc1..ff4e49a 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorMojoIT.java @@ -36,48 +36,47 @@ import java.util.List; import static org.apache.streams.plugins.test.StreamsHbaseResourceGeneratorTest.txtFilter; /** - * Tests that streams-plugin-hive running via maven generates hql resources + * Tests that streams-plugin-hbase running via maven generates txt resources. */ public class StreamsHbaseResourceGeneratorMojoIT extends TestCase { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoIT.class); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorMojoIT.class); - protected void setUp() throws Exception - { - // required for mojo lookups to work - super.setUp(); - } + protected void setUp() throws Exception { + // required for mojo lookups to work + super.setUp(); + } - @Test - public void testStreamsHbaseResourceGeneratorMojo() throws Exception { + @Test + public void testStreamsHbaseResourceGeneratorMojo() throws Exception { - File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" ); + File testDir = ResourceExtractor.simpleExtractResources( getClass(), "/streams-plugin-hbase" ); - Verifier verifier; + Verifier verifier; - verifier = new Verifier( testDir.getAbsolutePath() ); + verifier = new Verifier( testDir.getAbsolutePath() ); - List cliOptions = new ArrayList(); - cliOptions.add( "-N" ); - verifier.executeGoals( Lists.<String>newArrayList( - "clean", - "dependency:unpack-dependencies", - "generate-resources")); + List cliOptions = new ArrayList(); + cliOptions.add( "-N" ); + verifier.executeGoals( Lists.<String>newArrayList( + "clean", + "dependency:unpack-dependencies", + "generate-resources")); - verifier.verifyErrorFreeLog(); + verifier.verifyErrorFreeLog(); - verifier.resetStreams(); + verifier.resetStreams(); - File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo"); + File testOutput = new File(testDir.getAbsolutePath() + "/target/generated-resources/hbase-mojo"); - assert( testOutput != null ); - assert( testOutput.exists() == true ); - assert( testOutput.isDirectory() == true ); + assert ( testOutput != null ); + assert ( testOutput.exists() == true ); + assert ( testOutput.isDirectory() == true ); - Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) - .filter(txtFilter); - Collection<File> outputCollection = Lists.newArrayList(outputIterator); - assert( outputCollection.size() == 133 ); - } + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert ( outputCollection.size() == 133 ); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java index 254578c..ac876d5 100644 --- a/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java +++ b/streams-plugins/streams-plugin-hbase/src/test/java/org/apache/streams/plugins/test/StreamsHbaseResourceGeneratorTest.java @@ -19,22 +19,23 @@ package org.apache.streams.plugins.test; +import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig; +import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; + import com.google.common.base.Predicate; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.io.Files; import org.apache.commons.io.FileUtils; -import org.apache.streams.plugins.hbase.StreamsHbaseGenerationConfig; -import org.apache.streams.plugins.hbase.StreamsHbaseResourceGenerator; import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.annotation.Nullable; import java.io.File; import java.util.Collection; import java.util.Iterator; +import javax.annotation.Nullable; import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; @@ -43,79 +44,81 @@ import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; */ public class StreamsHbaseResourceGeneratorTest { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class); - - public static final Predicate<File> txtFilter = new Predicate<File>() { - @Override - public boolean apply(@Nullable File file) { - if( file.getName().endsWith(".txt") ) - return true; - else return false; - } - }; + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHbaseResourceGeneratorTest.class); - /** - * Tests that all example activities can be loaded into Activity beans - * - * @throws Exception - */ - @Test - public void StreamsHbaseResourceGenerator() throws Exception { + public static final Predicate<File> txtFilter = new Predicate<File>() { + @Override + public boolean apply(@Nullable File file) { + if ( file.getName().endsWith(".txt") ) { + return true; + } else { + return false; + } + } + }; - StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); + /** + * Tests that all example activities can be loaded into Activity beans. + * + * @throws Exception Exception + */ + @Test + public void testStreamsHbaseResourceGenerator() throws Exception { - String sourceDirectory = "target/test-classes/activitystreams-schemas"; + StreamsHbaseGenerationConfig config = new StreamsHbaseGenerationConfig(); - config.setSourceDirectory(sourceDirectory); + String sourceDirectory = "target/test-classes/activitystreams-schemas"; - config.setTargetDirectory("target/generated-resources/hbase"); + config.setSourceDirectory(sourceDirectory); - config.setExclusions(Sets.newHashSet("attachments")); + config.setTargetDirectory("target/generated-resources/hbase"); - config.setColumnFamily("cf"); - config.setMaxDepth(2); + config.setExclusions(Sets.newHashSet("attachments")); - StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); - streamsHbaseResourceGenerator.run(); + config.setColumnFamily("cf"); + config.setMaxDepth(2); - File testOutput = config.getTargetDirectory(); + StreamsHbaseResourceGenerator streamsHbaseResourceGenerator = new StreamsHbaseResourceGenerator(config); + streamsHbaseResourceGenerator.run(); - assert( testOutput != null ); - assert( testOutput.exists() == true ); - assert( testOutput.isDirectory() == true ); + File testOutput = config.getTargetDirectory(); - Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) - .filter(txtFilter); - Collection<File> outputCollection = Lists.newArrayList(outputIterator); - assert( outputCollection.size() == 133 ); + assert ( testOutput != null ); + assert ( testOutput.exists() == true ); + assert ( testOutput.isDirectory() == true ); - String expectedDirectory = "target/test-classes/expected"; - File testExpected = new File( expectedDirectory ); + Iterable<File> outputIterator = Files.fileTreeTraverser().breadthFirstTraversal(testOutput) + .filter(txtFilter); + Collection<File> outputCollection = Lists.newArrayList(outputIterator); + assert ( outputCollection.size() == 133 ); - Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) - .filter(txtFilter); - Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); + String expectedDirectory = "target/test-classes/expected"; + File testExpected = new File( expectedDirectory ); - int fails = 0; + Iterable<File> expectedIterator = Files.fileTreeTraverser().breadthFirstTraversal(testExpected) + .filter(txtFilter); + Collection<File> expectedCollection = Lists.newArrayList(expectedIterator); - Iterator<File> iterator = expectedCollection.iterator(); - while( iterator.hasNext() ) { - File objectExpected = iterator.next(); - String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); - File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); - LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); - assert( objectActual.exists()); - if( FileUtils.contentEquals(objectActual, objectExpected) == true ) { - LOGGER.info("Exact Match!"); - } else { - LOGGER.info("No Match!"); - fails++; - } - } - if( fails > 0 ) { - LOGGER.info("Fails: {}", fails); - Assert.fail(); - } + int fails = 0; + Iterator<File> iterator = expectedCollection.iterator(); + while ( iterator.hasNext() ) { + File objectExpected = iterator.next(); + String expectedEnd = dropSourcePathPrefix(objectExpected.getAbsolutePath(), expectedDirectory); + File objectActual = new File(config.getTargetDirectory() + "/" + expectedEnd); + LOGGER.info("Comparing: {} and {}", objectExpected.getAbsolutePath(), objectActual.getAbsolutePath()); + assert ( objectActual.exists()); + if ( FileUtils.contentEquals(objectActual, objectExpected) == true ) { + LOGGER.info("Exact Match!"); + } else { + LOGGER.info("No Match!"); + fails++; + } } + if ( fails > 0 ) { + LOGGER.info("Fails: {}", fails); + Assert.fail(); + } + + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java index b09ce18..cc32255 100644 --- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveGenerationConfig.java @@ -20,6 +20,7 @@ package org.apache.streams.plugins.hive; import org.apache.streams.util.schema.GenerationConfig; + import org.jsonschema2pojo.DefaultGenerationConfig; import org.jsonschema2pojo.util.URLUtil; @@ -33,70 +34,69 @@ import java.util.List; import java.util.Set; /** - * Configures StreamsHiveResourceGenerator - * - * + * Configures StreamsHiveResourceGenerator. */ public class StreamsHiveGenerationConfig extends DefaultGenerationConfig implements GenerationConfig { - public String getSourceDirectory() { - return sourceDirectory; - } + public String getSourceDirectory() { + return sourceDirectory; + } - public List<String> getSourcePaths() { - return sourcePaths; - } + public List<String> getSourcePaths() { + return sourcePaths; + } - private String sourceDirectory; - private List<String> sourcePaths = new ArrayList<String>(); - private String targetDirectory; - private int maxDepth = 1; + private String sourceDirectory; + private List<String> sourcePaths = new ArrayList<String>(); + private String targetDirectory; + private int maxDepth = 1; - public Set<String> getExclusions() { - return exclusions; - } + public Set<String> getExclusions() { + return exclusions; + } - public void setExclusions(Set<String> exclusions) { - this.exclusions = exclusions; - } + public void setExclusions(Set<String> exclusions) { + this.exclusions = exclusions; + } - private Set<String> exclusions = new HashSet<String>(); + private Set<String> exclusions = new HashSet<String>(); - public int getMaxDepth() { - return maxDepth; - } + public int getMaxDepth() { + return maxDepth; + } - public void setSourceDirectory(String sourceDirectory) { - this.sourceDirectory = sourceDirectory; - } + public void setSourceDirectory(String sourceDirectory) { + this.sourceDirectory = sourceDirectory; + } - public void setSourcePaths(List<String> sourcePaths) { - this.sourcePaths = sourcePaths; - } + public void setSourcePaths(List<String> sourcePaths) { + this.sourcePaths = sourcePaths; + } - public void setTargetDirectory(String targetDirectory) { - this.targetDirectory = targetDirectory; - } + public void setTargetDirectory(String targetDirectory) { + this.targetDirectory = targetDirectory; + } - @Override - public File getTargetDirectory() { - return new File(targetDirectory); - } + @Override + public File getTargetDirectory() { + return new File(targetDirectory); + } - @Override - public Iterator<URL> getSource() { - if (null != sourceDirectory) { - return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); - } - List<URL> sourceURLs = new ArrayList<URL>(); - if( sourcePaths != null && sourcePaths.size() > 0) - for (String source : sourcePaths) { - sourceURLs.add(URLUtil.parseURL(source)); - } - return sourceURLs.iterator(); + @Override + public Iterator<URL> getSource() { + if (null != sourceDirectory) { + return Collections.singleton(URLUtil.parseURL(sourceDirectory)).iterator(); } - - public void setMaxDepth(int maxDepth) { - this.maxDepth = maxDepth; + List<URL> sourceUrls = new ArrayList<URL>(); + if ( sourcePaths != null && sourcePaths.size() > 0) { + for (String source : sourcePaths) { + sourceUrls.add(URLUtil.parseURL(source)); + } } + return sourceUrls.iterator(); + } + + public void setMaxDepth(int maxDepth) { + this.maxDepth = maxDepth; + } } http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/5dffd5c3/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java ---------------------------------------------------------------------- diff --git a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java index c270e14..e6cb54c 100644 --- a/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java +++ b/streams-plugins/streams-plugin-hive/src/main/java/org/apache/streams/plugins/hive/StreamsHiveResourceGenerator.java @@ -19,12 +19,6 @@ package org.apache.streams.plugins.hive; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; import org.apache.streams.util.schema.FieldType; import org.apache.streams.util.schema.FieldUtil; import org.apache.streams.util.schema.FileUtil; @@ -32,17 +26,29 @@ import org.apache.streams.util.schema.GenerationConfig; import org.apache.streams.util.schema.Schema; import org.apache.streams.util.schema.SchemaStore; import org.apache.streams.util.schema.SchemaStoreImpl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; import org.jsonschema2pojo.util.URLUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.net.URL; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.commons.lang3.StringUtils.defaultString; -import static org.apache.streams.util.schema.FileUtil.*; +import static org.apache.streams.util.schema.FileUtil.dropExtension; +import static org.apache.streams.util.schema.FileUtil.dropSourcePathPrefix; +import static org.apache.streams.util.schema.FileUtil.swapExtension; +import static org.apache.streams.util.schema.FileUtil.writeFile; /** * Generates hive table definitions for using org.openx.data.jsonserde.JsonSerDe on new-line delimited json documents. @@ -51,277 +57,300 @@ import static org.apache.streams.util.schema.FileUtil.*; */ public class StreamsHiveResourceGenerator implements Runnable { - private final static Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGenerator.class); - - private final static String LS = System.getProperty("line.separator"); + private static final Logger LOGGER = LoggerFactory.getLogger(StreamsHiveResourceGenerator.class); - private StreamsHiveGenerationConfig config; + private static final String LS = System.getProperty("line.separator"); - private SchemaStore schemaStore = new SchemaStoreImpl(); + private StreamsHiveGenerationConfig config; - private int currentDepth = 0; + private SchemaStore schemaStore = new SchemaStoreImpl(); - public static void main(String[] args) { - StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig(); + private int currentDepth = 0; - String sourceDirectory = "src/main/jsonschema"; - String targetDirectory = "target/generated-resources/hive"; + /** + * Run from CLI without Maven + * + * <p/> + * java -jar streams-plugin-hive-jar-with-dependencies.jar StreamsHiveResourceGenerator src/main/jsonschema target/generated-resources + * + * @param args [sourceDirectory, targetDirectory] + * */ + public static void main(String[] args) { + StreamsHiveGenerationConfig config = new StreamsHiveGenerationConfig(); - if( args.length > 0 ) - sourceDirectory = args[0]; - if( args.length > 1 ) - targetDirectory = args[1]; + String sourceDirectory = "src/main/jsonschema"; + String targetDirectory = "target/generated-resources/hive"; - config.setSourceDirectory(sourceDirectory); - config.setTargetDirectory(targetDirectory); - - StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config); - streamsHiveResourceGenerator.run(); + if ( args.length > 0 ) { + sourceDirectory = args[0]; } - - public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) { - this.config = config; + if ( args.length > 1 ) { + targetDirectory = args[1]; } - public void run() { + config.setSourceDirectory(sourceDirectory); + config.setTargetDirectory(targetDirectory); - checkNotNull(config); + StreamsHiveResourceGenerator streamsHiveResourceGenerator = new StreamsHiveResourceGenerator(config); + streamsHiveResourceGenerator.run(); + } - generate(config); + public StreamsHiveResourceGenerator(StreamsHiveGenerationConfig config) { + this.config = config; + } - } + @Override + public void run() { - public void generate(StreamsHiveGenerationConfig config) { + checkNotNull(config); - LinkedList<File> sourceFiles = new LinkedList<File>(); + generate(config); - for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { - URL source = sources.next(); - sourceFiles.add(URLUtil.getFileFromURL(source)); - } + } - LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); + /** + * run generate using supplied StreamsHiveGenerationConfig. + * @param config StreamsHiveGenerationConfig + */ + public void generate(StreamsHiveGenerationConfig config) { - FileUtil.resolveRecursive((GenerationConfig)config, sourceFiles); + LinkedList<File> sourceFiles = new LinkedList<File>(); - LOGGER.info("Resolved {} schema files:", sourceFiles.size()); + for (Iterator<URL> sources = config.getSource(); sources.hasNext();) { + URL source = sources.next(); + sourceFiles.add(URLUtil.getFileFromURL(source)); + } - for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { - File item = iterator.next(); - schemaStore.create(item.toURI()); - } + LOGGER.info("Seeded with {} source paths:", sourceFiles.size()); - LOGGER.info("Identified {} objects:", schemaStore.getSize()); + FileUtil.resolveRecursive((GenerationConfig)config, sourceFiles); - for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { - Schema schema = schemaIterator.next(); - currentDepth = 0; - if( schema.getURI().getScheme().equals("file")) { - String inputFile = schema.getURI().getPath(); - String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); - for (String sourcePath : config.getSourcePaths()) { - resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); - } - String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "hql"); + LOGGER.info("Resolved {} schema files:", sourceFiles.size()); - LOGGER.info("Processing {}:", resourcePath); + for (Iterator<File> iterator = sourceFiles.iterator(); iterator.hasNext();) { + File item = iterator.next(); + schemaStore.create(item.toURI()); + } - String resourceId = dropExtension(resourcePath).replace("/", "_"); + LOGGER.info("Identified {} objects:", schemaStore.getSize()); - String resourceContent = generateResource(schema, resourceId); + for (Iterator<Schema> schemaIterator = schemaStore.getSchemaIterator(); schemaIterator.hasNext(); ) { + Schema schema = schemaIterator.next(); + currentDepth = 0; + if ( schema.getUri().getScheme().equals("file")) { + String inputFile = schema.getUri().getPath(); + String resourcePath = dropSourcePathPrefix(inputFile, config.getSourceDirectory()); + for (String sourcePath : config.getSourcePaths()) { + resourcePath = dropSourcePathPrefix(resourcePath, sourcePath); + } + String outputFile = config.getTargetDirectory() + "/" + swapExtension(resourcePath, "json", "hql"); - writeFile(outputFile, resourceContent); + LOGGER.info("Processing {}:", resourcePath); - LOGGER.info("Wrote {}:", outputFile); - } - } - } + String resourceId = dropExtension(resourcePath).replace("/", "_"); - public String generateResource(Schema schema, String resourceId) { - StringBuilder resourceBuilder = new StringBuilder(); - resourceBuilder.append("CREATE TABLE "); - resourceBuilder.append(hqlEscape(resourceId)); - resourceBuilder.append(LS); - resourceBuilder.append("("); - resourceBuilder.append(LS); - resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' '); - resourceBuilder.append(")"); - resourceBuilder.append(LS); - resourceBuilder.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'"); - resourceBuilder.append(LS); - resourceBuilder.append("WITH SERDEPROPERTIES (\"ignore.malformed.json\" = \"true\""); - resourceBuilder.append(LS); - resourceBuilder.append("STORED AS TEXTFILE"); - resourceBuilder.append(LS); - resourceBuilder.append("LOCATION '${hiveconf:path}';"); - resourceBuilder.append(LS); - return resourceBuilder.toString(); - } + String resourceContent = generateResource(schema, resourceId); - public StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) { - ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); - if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { - builder = appendPropertiesNode(builder, schema, propertiesNode, seperator); - } - return builder; - } + writeFile(outputFile, resourceContent); - private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { - // safe to append nothing - checkNotNull(builder); - builder.append(hqlEscape(fieldId)); - builder.append(seperator); - builder.append(hqlType(fieldType)); - return builder; + LOGGER.info("Wrote {}:", outputFile); + } } - - public StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) { - // not safe to append nothing - checkNotNull(builder); - if( itemsNode == null ) return builder; - if( itemsNode.has("type")) { - try { - FieldType itemType = FieldUtil.determineFieldType(itemsNode); - switch( itemType ) { - case OBJECT: - builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator); - break; - case ARRAY: - ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items"); - builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator); - break; - default: - builder = appendArrayField(builder, schema, fieldId, itemType, seperator); - } - } catch (Exception e) { - LOGGER.warn("No item type resolvable for {}", fieldId); - } - } - checkNotNull(builder); - return builder; + } + + /** + * generateResource String from schema and resourceId. + * @param schema Schema + * @param resourceId String + * @return CREATE TABLE ... + */ + public String generateResource(Schema schema, String resourceId) { + StringBuilder resourceBuilder = new StringBuilder(); + resourceBuilder.append("CREATE TABLE "); + resourceBuilder.append(hqlEscape(resourceId)); + resourceBuilder.append(LS); + resourceBuilder.append("("); + resourceBuilder.append(LS); + resourceBuilder = appendRootObject(resourceBuilder, schema, resourceId, ' '); + resourceBuilder.append(")"); + resourceBuilder.append(LS); + resourceBuilder.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'"); + resourceBuilder.append(LS); + resourceBuilder.append("WITH SERDEPROPERTIES (\"ignore.malformed.json\" = \"true\""); + resourceBuilder.append(LS); + resourceBuilder.append("STORED AS TEXTFILE"); + resourceBuilder.append(LS); + resourceBuilder.append("LOCATION '${hiveconf:path}';"); + resourceBuilder.append(LS); + return resourceBuilder.toString(); + } + + protected StringBuilder appendRootObject(StringBuilder builder, Schema schema, String resourceId, Character seperator) { + ObjectNode propertiesNode = schemaStore.resolveProperties(schema, null, resourceId); + if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0) { + builder = appendPropertiesNode(builder, schema, propertiesNode, seperator); } - - private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { - // safe to append nothing - checkNotNull(builder); - checkNotNull(fieldId); - builder.append(hqlEscape(fieldId)); - builder.append(seperator); - builder.append("ARRAY<"+hqlType(fieldType)+">"); - checkNotNull(builder); - return builder; + return builder; + } + + private StringBuilder appendValueField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { + // safe to append nothing + checkNotNull(builder); + builder.append(hqlEscape(fieldId)); + builder.append(seperator); + builder.append(hqlType(fieldType)); + return builder; + } + + protected StringBuilder appendArrayItems(StringBuilder builder, Schema schema, String fieldId, ObjectNode itemsNode, Character seperator) { + // not safe to append nothing + checkNotNull(builder); + if ( itemsNode == null ) { + return builder; } - - private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) { - // safe to append nothing - checkNotNull(builder); - checkNotNull(fieldNode); - if( !Strings.isNullOrEmpty(fieldId)) { - builder.append(hqlEscape(fieldId)); - builder.append(seperator); + if ( itemsNode.has("type")) { + try { + FieldType itemType = FieldUtil.determineFieldType(itemsNode); + switch ( itemType ) { + case OBJECT: + builder = appendArrayObject(builder, schema, fieldId, itemsNode, seperator); + break; + case ARRAY: + ObjectNode subArrayItems = (ObjectNode) itemsNode.get("items"); + builder = appendArrayItems(builder, schema, fieldId, subArrayItems, seperator); + break; + default: + builder = appendArrayField(builder, schema, fieldId, itemType, seperator); } - builder.append("ARRAY"); - builder.append(LS); - builder.append("<"); - builder.append(LS); - ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId); - builder = appendStructField(builder, schema, "", propertiesNode, ':'); - builder.append(">"); - checkNotNull(builder); - return builder; + } catch (Exception ex) { + LOGGER.warn("No item type resolvable for {}", fieldId); + } } + checkNotNull(builder); + return builder; + } + + private StringBuilder appendArrayField(StringBuilder builder, Schema schema, String fieldId, FieldType fieldType, Character seperator) { + // safe to append nothing + checkNotNull(builder); + checkNotNull(fieldId); + builder.append(hqlEscape(fieldId)); + builder.append(seperator); + builder.append("ARRAY<" + hqlType(fieldType) + ">"); + checkNotNull(builder); + return builder; + } + + private StringBuilder appendArrayObject(StringBuilder builder, Schema schema, String fieldId, ObjectNode fieldNode, Character seperator) { + // safe to append nothing + checkNotNull(builder); + checkNotNull(fieldNode); + if ( !Strings.isNullOrEmpty(fieldId)) { + builder.append(hqlEscape(fieldId)); + builder.append(seperator); + } + builder.append("ARRAY"); + builder.append(LS); + builder.append("<"); + builder.append(LS); + ObjectNode propertiesNode = schemaStore.resolveProperties(schema, fieldNode, fieldId); + builder = appendStructField(builder, schema, "", propertiesNode, ':'); + builder.append(">"); + checkNotNull(builder); + return builder; + } + + private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) { + // safe to append nothing + checkNotNull(builder); + checkNotNull(propertiesNode); + + if ( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) { + + currentDepth += 1; + + if ( !Strings.isNullOrEmpty(fieldId)) { + builder.append(hqlEscape(fieldId)); + builder.append(seperator); + } + builder.append("STRUCT"); + builder.append(LS); + builder.append("<"); + builder.append(LS); - private StringBuilder appendStructField(StringBuilder builder, Schema schema, String fieldId, ObjectNode propertiesNode, Character seperator) { - // safe to append nothing - checkNotNull(builder); - checkNotNull(propertiesNode); - - if( propertiesNode != null && propertiesNode.isObject() && propertiesNode.size() > 0 ) { - - currentDepth += 1; - - if( !Strings.isNullOrEmpty(fieldId)) { - builder.append(hqlEscape(fieldId)); - builder.append(seperator); - } - builder.append("STRUCT"); - builder.append(LS); - builder.append("<"); - builder.append(LS); - - builder = appendPropertiesNode(builder, schema, propertiesNode, ':'); + builder = appendPropertiesNode(builder, schema, propertiesNode, ':'); - builder.append(">"); - builder.append(LS); + builder.append(">"); + builder.append(LS); - currentDepth -= 1; + currentDepth -= 1; - } - checkNotNull(builder); - return builder; } - - private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) { - checkNotNull(builder); - checkNotNull(propertiesNode); - Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); - Joiner joiner = Joiner.on(","+LS).skipNulls(); - List<String> fieldStrings = Lists.newArrayList(); - for( ; fields.hasNext(); ) { - Map.Entry<String, JsonNode> field = fields.next(); - String fieldId = field.getKey(); - if( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { - ObjectNode fieldNode = (ObjectNode) field.getValue(); - FieldType fieldType = FieldUtil.determineFieldType(fieldNode); - if (fieldType != null ) { - switch (fieldType) { - case ARRAY: - ObjectNode itemsNode = (ObjectNode) fieldNode.get("items"); - if( currentDepth <= config.getMaxDepth()) { - StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator); - if( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) { - fieldStrings.add(arrayItemsBuilder.toString()); - } - } - break; - case OBJECT: - ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId); - if( currentDepth < config.getMaxDepth()) { - StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator); - if( !Strings.isNullOrEmpty(structFieldBuilder.toString())) { - fieldStrings.add(structFieldBuilder.toString()); - } - } - break; - default: - StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator); - if( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) { - fieldStrings.add(valueFieldBuilder.toString()); - } - } + checkNotNull(builder); + return builder; + } + + private StringBuilder appendPropertiesNode(StringBuilder builder, Schema schema, ObjectNode propertiesNode, Character seperator) { + checkNotNull(builder); + checkNotNull(propertiesNode); + Iterator<Map.Entry<String, JsonNode>> fields = propertiesNode.fields(); + Joiner joiner = Joiner.on("," + LS).skipNulls(); + List<String> fieldStrings = Lists.newArrayList(); + for ( ; fields.hasNext(); ) { + Map.Entry<String, JsonNode> field = fields.next(); + String fieldId = field.getKey(); + if ( !config.getExclusions().contains(fieldId) && field.getValue().isObject()) { + ObjectNode fieldNode = (ObjectNode) field.getValue(); + FieldType fieldType = FieldUtil.determineFieldType(fieldNode); + if (fieldType != null ) { + switch (fieldType) { + case ARRAY: + ObjectNode itemsNode = (ObjectNode) fieldNode.get("items"); + if ( currentDepth <= config.getMaxDepth()) { + StringBuilder arrayItemsBuilder = appendArrayItems(new StringBuilder(), schema, fieldId, itemsNode, seperator); + if ( !Strings.isNullOrEmpty(arrayItemsBuilder.toString())) { + fieldStrings.add(arrayItemsBuilder.toString()); } - } - } - builder.append(joiner.join(fieldStrings)).append(LS); - Preconditions.checkNotNull(builder); - return builder; - } - - private static String hqlEscape( String fieldId ) { - return "`"+fieldId+"`"; - } - - private static String hqlType( FieldType fieldType ) { - switch( fieldType ) { - case INTEGER: - return "INT"; - case NUMBER: - return "FLOAT"; + } + break; case OBJECT: - return "STRUCT"; + ObjectNode childProperties = schemaStore.resolveProperties(schema, fieldNode, fieldId); + if ( currentDepth < config.getMaxDepth()) { + StringBuilder structFieldBuilder = appendStructField(new StringBuilder(), schema, fieldId, childProperties, seperator); + if ( !Strings.isNullOrEmpty(structFieldBuilder.toString())) { + fieldStrings.add(structFieldBuilder.toString()); + } + } + break; default: - return fieldType.name().toUpperCase(); + StringBuilder valueFieldBuilder = appendValueField(new StringBuilder(), schema, fieldId, fieldType, seperator); + if ( !Strings.isNullOrEmpty(valueFieldBuilder.toString())) { + fieldStrings.add(valueFieldBuilder.toString()); + } + } } + } + } + builder.append(joiner.join(fieldStrings)).append(LS); + Preconditions.checkNotNull(builder); + return builder; + } + + private static String hqlEscape( String fieldId ) { + return "`" + fieldId + "`"; + } + + private static String hqlType( FieldType fieldType ) { + switch ( fieldType ) { + case INTEGER: + return "INT"; + case NUMBER: + return "FLOAT"; + case OBJECT: + return "STRUCT"; + default: + return fieldType.name().toUpperCase(); } + } }