Repository: incubator-hawq Updated Branches: refs/heads/master c0d7c4fdb -> fd9c36861
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserSeekTest.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserSeekTest.java b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserSeekTest.java new file mode 100644 index 0000000..f42f653 --- /dev/null +++ b/pxf/pxf-json/src/test/java/org/apache/hawq/pxf/plugins/json/parser/PartitionedJsonParserSeekTest.java @@ -0,0 +1,113 @@ +package org.apache.hawq.pxf.plugins.json.parser; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Comparator; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; + +public class PartitionedJsonParserSeekTest { + + private static final Log LOG = LogFactory.getLog(PartitionedJsonParserSeekTest.class); + + @Test + public void testNoSeek() throws IOException { + File testsDir = new File("src/test/resources/parser-tests/seek"); + File[] dirs = testsDir.listFiles(); + + for (File jsonDir : dirs) { + runTest(jsonDir); + } + } + + public void runTest(final File jsonDir) throws IOException { + + File jsonFile = new File(jsonDir, "input.json"); + InputStream jsonInputStream = new FileInputStream(jsonFile); + + try { + seekToStart(jsonInputStream); + PartitionedJsonParser parser = new PartitionedJsonParser(jsonInputStream); + + File[] jsonOjbectFiles = jsonFile.getParentFile().listFiles(new FilenameFilter() { + public boolean accept(File file, String s) { + return s.contains("expected"); + } + }); + + Arrays.sort(jsonOjbectFiles, new Comparator<File>() { + public int compare(File file, File file1) { + return file.compareTo(file1); + } + }); + + if (jsonOjbectFiles == null || jsonOjbectFiles.length == 0) { + String result = parser.nextObjectContainingMember("name"); + assertNull("File " + jsonFile.getAbsolutePath() + " got result '" + result + "'", result); + LOG.info("File " + jsonFile.getAbsolutePath() + " passed"); + } else { + for (File jsonObjectFile : jsonOjbectFiles) { + String expected = trimWhitespaces(FileUtils.readFileToString(jsonObjectFile)); + String result = parser.nextObjectContainingMember("name"); + assertNotNull(jsonFile.getAbsolutePath() + "/" + jsonObjectFile.getName(), result); + assertEquals(jsonFile.getAbsolutePath() + "/" + jsonObjectFile.getName(), expected, + trimWhitespaces(result)); + LOG.info("File " + jsonFile.getAbsolutePath() + "/" + jsonObjectFile.getName() + " passed"); + } + } + + } finally { + IOUtils.closeQuietly(jsonInputStream); + } + } + + public void seekToStart(InputStream jsonInputStream) throws IOException { + // pop off characters until we see <SEEK> + StringBuilder sb = new StringBuilder(); + int i; + while ((i = jsonInputStream.read()) != -1) { + sb.append((char) i); + + if (sb.toString().endsWith("<SEEK>")) { + return; + } + } + assertTrue(false); + } + + public String trimWhitespaces(String s) { + return s.replaceAll("[\\n\\t\\r \\t]+", " ").trim(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/datatypes-test.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/datatypes-test.json b/pxf/pxf-json/src/test/resources/datatypes-test.json new file mode 100644 index 0000000..4a08ff9 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/datatypes-test.json @@ -0,0 +1,14 @@ +{ + "bintType" : 666, + "booleanType" : "true", + "charType" : "x", + "byteaType" : "a", + "float8Type" : 3.14, + "realType" : 3.15, + "integerType" :999, + "smallintType": 777, + "bpcharType" :"bpcharType", + "varcharType" :"varcharType", + "textType" : "textType" + } + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json new file mode 100644 index 0000000..64b5b73 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json @@ -0,0 +1,24 @@ +[ + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json.state new file mode 100644 index 0000000..8d4057c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_complex.json.state @@ -0,0 +1,149 @@ +BEGIN_ARRAY +BEGIN_OBJECT + +# color=red +BEGIN_STRING +INSIDE_STRING{5} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{3} +END_STRING + +VALUE_SEPARATOR + +# "v"="vv" +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{2} +END_STRING + +VALUE_SEPARATOR + +# "a"=true +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{4} + +VALUE_SEPARATOR + +# "b"=false +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{5} + +VALUE_SEPARATOR + +# "c"=123.45 +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{6} + +VALUE_SEPARATOR + +# "d"=null +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{4} + +VALUE_SEPARATOR + +# "e"={... +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR + +BEGIN_OBJECT + +# "e1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# ""="" +BEGIN_STRING +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +# end e +END_OBJECT + +VALUE_SEPARATOR + +# "f"=[{... +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR + +BEGIN_ARRAY +BEGIN_OBJECT + +# "f1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# "f2"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +END_OBJECT + +VALUE_SEPARATOR + +BEGIN_OBJECT + +# "f1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# "f2"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +END_OBJECT + +END_ARRAY + +END_OBJECT + +END_ARRAY \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json new file mode 100644 index 0000000..6c28132 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json @@ -0,0 +1 @@ +[{},{}] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json.state new file mode 100644 index 0000000..06ed04e --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_objects_empty.json.state @@ -0,0 +1,7 @@ +BEGIN_ARRAY +BEGIN_OBJECT +END_OBJECT +VALUE_SEPARATOR +BEGIN_OBJECT +END_OBJECT +END_ARRAY \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json b/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json new file mode 100644 index 0000000..1962b0c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json @@ -0,0 +1 @@ +[ 100, 500 ] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json.state new file mode 100644 index 0000000..2a83fa9 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/array_of_numbers.json.state @@ -0,0 +1,5 @@ +BEGIN_ARRAY +DONT_CARE{3} +VALUE_SEPARATOR +DONT_CARE{3} +END_ARRAY \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json b/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json new file mode 100644 index 0000000..a0237d1 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json.state new file mode 100644 index 0000000..6e8d45b --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_complex.json.state @@ -0,0 +1,146 @@ +BEGIN_OBJECT + +# color=red +BEGIN_STRING +INSIDE_STRING{5} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{3} +END_STRING + +VALUE_SEPARATOR + +# "v"="vv" +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{2} +END_STRING + +VALUE_SEPARATOR + +# "a"=true +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{4} + +VALUE_SEPARATOR + +# "b"=false +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{5} + +VALUE_SEPARATOR + +# "c"=123.45 +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{6} + +VALUE_SEPARATOR + +# "d"=null +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR +DONT_CARE{4} + +VALUE_SEPARATOR + +# "e"={... +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR + +BEGIN_OBJECT + +# "e1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# ""="" +BEGIN_STRING +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +# end e +END_OBJECT + +VALUE_SEPARATOR + +# "f"=[{... +BEGIN_STRING +INSIDE_STRING +END_STRING +NAME_SEPARATOR + +BEGIN_ARRAY +BEGIN_OBJECT + +# "f1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# "f2"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +END_OBJECT + +VALUE_SEPARATOR + +BEGIN_OBJECT + +# "f1"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +VALUE_SEPARATOR + +# "f2"="" +BEGIN_STRING +INSIDE_STRING{2} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +END_STRING + +END_OBJECT + +END_ARRAY + +END_OBJECT http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json b/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json new file mode 100644 index 0000000..c964b01 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json @@ -0,0 +1,4 @@ +{ + "color": "red", + "value": "#f00" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json.state new file mode 100644 index 0000000..0c16ea8 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_simple.json.state @@ -0,0 +1,24 @@ +BEGIN_OBJECT + +# "color": "red" +BEGIN_STRING +INSIDE_STRING{5} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{3} +END_STRING + +VALUE_SEPARATOR + +# "value": "#f00" +BEGIN_STRING +INSIDE_STRING{5} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{4} +END_STRING + + +END_OBJECT http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json b/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json new file mode 100644 index 0000000..9da1c38 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json @@ -0,0 +1,4 @@ +{ + "i\"\\\/d": "0\b\f\n\r\t\u0644001", + "type": "donut" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json.state ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json.state b/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json.state new file mode 100644 index 0000000..919124d --- /dev/null +++ b/pxf/pxf-json/src/test/resources/lexer-tests/object_string_escaping.json.state @@ -0,0 +1,44 @@ +BEGIN_OBJECT + +# "i\"\\\/d": ... +BEGIN_STRING +INSIDE_STRING +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{2} +END_STRING + +NAME_SEPARATOR + +# "0\b\f\n\r\t\u0644001", +BEGIN_STRING +INSIDE_STRING +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{1} +STRING_ESCAPE +INSIDE_STRING{8} +END_STRING + +VALUE_SEPARATOR + +# "type": "donut" +BEGIN_STRING +INSIDE_STRING{4} +END_STRING +NAME_SEPARATOR +BEGIN_STRING +INSIDE_STRING{5} +END_STRING + +END_OBJECT http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/log4j.properties b/pxf/pxf-json/src/test/resources/log4j.properties new file mode 100644 index 0000000..bcdc149 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=DEBUG,INFO + +log4j.logger.org.apache.hawq.pxf=INFO +log4j.logger.org.apache.hawq.pxf.plugins.json=DEBUG + +log4j.logger.org.apache.hadoop=ERROR + +# A1 is set to be a ConsoleAppender. +log4j.appender.INFO=org.apache.log4j.ConsoleAppender +log4j.appender.ERROR=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.INFO.layout=org.apache.log4j.PatternLayout +log4j.appender.INFO.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +log4j.appender.ERROR.layout=org.apache.log4j.PatternLayout +log4j.appender.ERROR.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + + +# An alternative logging format: +# log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1} - %m%n \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/null-tweets.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/null-tweets.json b/pxf/pxf-json/src/test/resources/null-tweets.json new file mode 100644 index 0000000..38e65a2 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/null-tweets.json @@ -0,0 +1,2 @@ +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","text":"text1","source":"web","user":{"id":"26643566","name":"SpreadButter","screen_name":"SpreadButter","location":"Austin, Texas"},"entities":{"hashtags":["tweetCongress","IRS"]}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":null,"id_str":"343136547123646465","text":"text2","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","user":{"id":102904200,"id_str":"102904200","name":"Ardianto","screen_name":"patronusdeadly","location":"Bekasi-Surabaya"},"entities":{"hashtags":[]}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json new file mode 100644 index 0000000..1eda5c8 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json @@ -0,0 +1,24 @@ +[ + { + "color": "red", + "v": "vv", + "a": true, + "name": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json.expected1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json.expected1.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json.expected1.json new file mode 100644 index 0000000..0c48a9b --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex.json.expected1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "name": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json new file mode 100644 index 0000000..e7fca84 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json @@ -0,0 +1,24 @@ +[ + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "name":"" + }, + { + "name":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected1.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected1.json new file mode 100644 index 0000000..d169f64 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected1.json @@ -0,0 +1,4 @@ +{ + "f1":"", + "name":"" + } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected2.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected2.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected2.json new file mode 100644 index 0000000..4a56873 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_complex2.json.expected2.json @@ -0,0 +1,4 @@ +{ + "name":"", + "f2":"" + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json new file mode 100644 index 0000000..f60d588 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json @@ -0,0 +1,24 @@ +[ + { + "color": "red", + "v": "vv", + "a": true, + "name": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "name":"", + "f2":"" + }, + { + "name":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json.expected1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json.expected1.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json.expected1.json new file mode 100644 index 0000000..2fc7674 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/array_objects_same_name_in_child.json.expected1.json @@ -0,0 +1,22 @@ + { + "color": "red", + "v": "vv", + "a": true, + "name": false, + "c": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "name":"", + "f2":"" + }, + { + "name":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json new file mode 100644 index 0000000..8de19e8 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json @@ -0,0 +1,4 @@ +{ + "name": "red", + "value": "#f00" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json.expected1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json.expected1.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json.expected1.json new file mode 100644 index 0000000..8de19e8 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_simple.json.expected1.json @@ -0,0 +1,4 @@ +{ + "name": "red", + "value": "#f00" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json new file mode 100644 index 0000000..191ffd2 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json @@ -0,0 +1,4 @@ +{ + "i\"\\\/d": "0\b\f\n\r\t\u0644001", + "name": "donut" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json.expected1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json.expected1.json b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json.expected1.json new file mode 100644 index 0000000..191ffd2 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/noseek/object_string_escaping.json.expected1.json @@ -0,0 +1,4 @@ +{ + "i\"\\\/d": "0\b\f\n\r\t\u0644001", + "name": "donut" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/expected.1.json new file mode 100644 index 0000000..679fd78 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/expected.1.json @@ -0,0 +1,44 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ], + + "name": 123.45 + + , + "g": [ + { + "f1":"", + "f2":{ + "f11": [ + { + "f12": { + } + }, + {}, + {} + ] + } + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/input.json new file mode 100644 index 0000000..2c23476 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member/input.json @@ -0,0 +1,59 @@ +[ + { + "f": [ + { <SEEK> + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } + , + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ], + + "name": 123.45 + + , + "g": [ + { + "f1":"", + "f2":{ + "f11": [ + { + "f12": { + } + }, + {}, + {} + ] + } + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/expected.1.json new file mode 100644 index 0000000..679fd78 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/expected.1.json @@ -0,0 +1,44 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ], + + "name": 123.45 + + , + "g": [ + { + "f1":"", + "f2":{ + "f11": [ + { + "f12": { + } + }, + {}, + {} + ] + } + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/input.json new file mode 100644 index 0000000..6aaa5d9 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/child-object-before-member2/input.json @@ -0,0 +1,59 @@ +[ + <SEEK> { + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } + , + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ], + + "name": 123.45 + + , + "g": [ + { + "f1":"", + "f2":{ + "f11": [ + { + "f12": { + } + }, + {}, + {} + ] + } + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.2.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.2.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.2.json new file mode 100644 index 0000000..2dd6d49 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/expected.2.json @@ -0,0 +1,25 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":[ + {"a":"b"}, + {"a":"b"} + ] + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/input.json new file mode 100644 index 0000000..fd72687 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/multi/input.json @@ -0,0 +1,48 @@ +[ <SEEK> + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":[ + {"a":"b"}, + {"a":"b"} + ] + }, + { + "f1":"", + "f2":"" + } + ] + }] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/no-elements/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/no-elements/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/no-elements/input.json new file mode 100644 index 0000000..4b1128f --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/no-elements/input.json @@ -0,0 +1,24 @@ +[ + { <SEEK> + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/input.json new file mode 100644 index 0000000..fa2c469 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-1/input.json @@ -0,0 +1,28 @@ +[ + { "asd1":null, + <SEEK> "name": "as\{d", + "asd2":null + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/input.json new file mode 100644 index 0000000..eb33030 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-2/input.json @@ -0,0 +1,28 @@ +[ + { "asd1":null, + "name": <SEEK> "as\{d", + "asd2":null + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/input.json new file mode 100644 index 0000000..9a4cb10 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-mid-object-3/input.json @@ -0,0 +1,28 @@ +[ + { "asd1":null, + "name": "as\{d", <SEEK> + "asd2":null + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/input.json new file mode 100644 index 0000000..e19a97e --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-1/input.json @@ -0,0 +1,27 @@ +[ + { "na<SEEK>me": "asd", + "asd":null + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/input.json new file mode 100644 index 0000000..a744b02 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/seek-into-string-2/input.json @@ -0,0 +1,27 @@ +[ + { "name": "a<SEEK>s\{d", + "asd":null + }, + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/expected.1.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/expected.1.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/expected.1.json new file mode 100644 index 0000000..2e5c05c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/expected.1.json @@ -0,0 +1,22 @@ +{ + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/input.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/input.json b/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/input.json new file mode 100644 index 0000000..1df5f9c --- /dev/null +++ b/pxf/pxf-json/src/test/resources/parser-tests/seek/simple/input.json @@ -0,0 +1,24 @@ +[ <SEEK> + { + "color": "red", + "v": "vv", + "a": true, + "b": false, + "name": 123.45, + "d": null, + "e": { + "e1":"", + "":"" + }, + "f": [ + { + "f1":"", + "f2":"" + }, + { + "f1":"", + "f2":"" + } + ] + } +] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/sample-malformed.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/sample-malformed.json b/pxf/pxf-json/src/test/resources/sample-malformed.json new file mode 100644 index 0000000..f886d0a --- /dev/null +++ b/pxf/pxf-json/src/test/resources/sample-malformed.json @@ -0,0 +1,3 @@ +{"menuitem":{"key1":"Foo"}} +{"menuitem":{"key1":"Bar"}} +{"menuitem":{"key1":"Baz"}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/sample.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/sample.json b/pxf/pxf-json/src/test/resources/sample.json new file mode 100644 index 0000000..f886d0a --- /dev/null +++ b/pxf/pxf-json/src/test/resources/sample.json @@ -0,0 +1,3 @@ +{"menuitem":{"key1":"Foo"}} +{"menuitem":{"key1":"Bar"}} +{"menuitem":{"key1":"Baz"}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-broken.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-broken.json b/pxf/pxf-json/src/test/resources/tweets-broken.json new file mode 100644 index 0000000..8d2c5b8 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-broken.json @@ -0,0 +1,67 @@ +{ + "root":[ + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547115253761, + "id_str":"343136547115253761", + "text":"text1", + "source":"<a href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>", + "user":{ + "id":26643566, + "name":"SpreadButter", + "screen_name":"SpreadButter", + "location":"Austin, Texas" + }, + "entities":{ + "hashtags":[ + "tweetCongress", + "IRS" + ] + } + } + }, + { + "record": + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547123646465, + "id_str":"343136547123646465", + "text":"text2", + "source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e", + "user":{ + "id":102904200, + "id_str":"102904200", + "name":"Ardianto", + "screen_name":"patronusdeadly", + "location":"Bekasi-Surabaya" + }, + "entities":{ + "hashtags":[ + + ] + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547136233472, + "id_str":"343136547136233472", + "text":"text3", + "source":"\u003ca href=\"http:\/\/www.nosecrets-empregos.com.br\/homologa\" rel=\"nofollow\"\u003eVagas NoSecrets\u003c\/a\u003e", + "user":{ + "id":287819058, + "id_str":"287819058", + "name":"No Secrets Empregos", + "screen_name":"NoSecrets_Vagas", + "location":"" + }, + "entities":{ + "hashtags":[ + + ] + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-pp-with-delete.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-pp-with-delete.json b/pxf/pxf-json/src/test/resources/tweets-pp-with-delete.json new file mode 100644 index 0000000..bfcf269 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-pp-with-delete.json @@ -0,0 +1,79 @@ +{ + "root":[ + { + "record": { + "delete": { + "status": { + "id": 322588945445691392, + "user_id": 775249976, + "id_str": "322588945445691392", + "user_id_str": "775249976" + } + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547115253761, + "id_str":"343136547115253761", + "text":"text1", + "source":"<a href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>", + "user":{ + "id":26643566, + "name":"SpreadButter", + "screen_name":"SpreadButter", + "location":"Austin, Texas" + }, + "entities":{ + "hashtags":[ + "tweetCongress", + "IRS" + ] + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547123646465, + "id_str":"343136547123646465", + "text":"text2", + "source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e", + "user":{ + "id":102904200, + "id_str":"102904200", + "name":"Ardianto", + "screen_name":"patronusdeadly", + "location":"Bekasi-Surabaya" + }, + "entities":{ + "hashtags":[ + + ] + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547136233472, + "id_str":"343136547136233472", + "text":"text3", + "source":"\u003ca href=\"http:\/\/www.nosecrets-empregos.com.br\/homologa\" rel=\"nofollow\"\u003eVagas NoSecrets\u003c\/a\u003e", + "user":{ + "id":287819058, + "id_str":"287819058", + "name":"No Secrets Empregos", + "screen_name":"NoSecrets_Vagas", + "location":"" + }, + "entities":{ + "hashtags":[ + + ] + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-pp.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-pp.json b/pxf/pxf-json/src/test/resources/tweets-pp.json new file mode 100644 index 0000000..c13abeb --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-pp.json @@ -0,0 +1,67 @@ +{ + "root":[ + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547115253761, + "id_str":"343136547115253761", + "text":"text1", + "source":"<a href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>", + "user":{ + "id":26643566, + "name":"SpreadButter", + "screen_name":"SpreadButter", + "location":"Austin, Texas" + }, + "entities":{ + "hashtags":[ + "tweetCongress", + "IRS" + ] + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547123646465, + "id_str":"343136547123646465", + "text":"text2", + "source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e", + "user":{ + "id":102904200, + "id_str":"102904200", + "name":"Ardianto", + "screen_name":"patronusdeadly", + "location":"Bekasi-Surabaya" + }, + "entities":{ + "hashtags":[ + + ] + } + } + }, + { + "record":{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547136233472, + "id_str":"343136547136233472", + "text":"text3", + "source":"\u003ca href=\"http:\/\/www.nosecrets-empregos.com.br\/homologa\" rel=\"nofollow\"\u003eVagas NoSecrets\u003c\/a\u003e", + "user":{ + "id":287819058, + "id_str":"287819058", + "name":"No Secrets Empregos", + "screen_name":"NoSecrets_Vagas", + "location":"" + }, + "entities":{ + "hashtags":[ + + ] + } + } + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-small-with-delete.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-small-with-delete.json b/pxf/pxf-json/src/test/resources/tweets-small-with-delete.json new file mode 100644 index 0000000..3e613a7 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-small-with-delete.json @@ -0,0 +1,4 @@ +{"delete":{"status":{"id":322588945445691392,"user_id":775249976,"id_str":"322588945445691392","user_id_str":"775249976"}}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547115253761,"id_str":"343136547115253761","text":"text1","source":"<a href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>","user":{"id":26643566,"name":"SpreadButter","screen_name":"SpreadButter","location":"Austin, Texas"},"entities":{"hashtags":["tweetCongress","IRS"]}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547123646465,"id_str":"343136547123646465","text":"text2","source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","user":{"id":102904200,"id_str":"102904200","name":"Ardianto","screen_name":"patronusdeadly","location":"Bekasi-Surabaya"},"entities":{"hashtags":[]}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547136233472,"id_str":"343136547136233472","text":"text3","source":"\u003ca href=\"http:\/\/www.nosecrets-empregos.com.br\/homologa\" rel=\"nofollow\"\u003eVagas NoSecrets\u003c\/a\u003e","user":{"id":287819058,"id_str":"287819058","name":"No Secrets Empregos","screen_name":"NoSecrets_Vagas","location":""},"entities":{"hashtags":[]}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-small.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-small.json b/pxf/pxf-json/src/test/resources/tweets-small.json new file mode 100644 index 0000000..cfbb4c5 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-small.json @@ -0,0 +1,4 @@ +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547115253761,"text":"text1","source":"web","user":{"id":"26643566","name":"SpreadButter","screen_name":"SpreadButter","location":"Austin, Texas"},"entities":{"hashtags":["tweetCongress","IRS"]}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547123646465,"text":"text2","source":"web","user":{"id":"102904200","name":"Ardianto","screen_name":"patronusdeadly","location":"Bekasi-Surabaya"},"entities":{"hashtags":[]}} +{"created_at":"Fri Jun 07 22:45:02 +0000 2013","id":343136547136233472,"text":"text3","source":"web","user":{"id":"287819058","screen_name":"NoSecrets_Vagas","location":""},"entities":{"hashtags":[]}} +{"created_at":"Fri Jun 07 22:45:03 +0000 2013","id":343136551322136576,"text":"text4","source":"web","user":{"id":"395504494","screen_name":"SevenStonesBuoy","location":"Near Cornwall"},"coordinates":{"type":"Point","coordinates":[-6.100,50.103]},"entities":{"hashtags":[]}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets-with-missing-text-attribtute.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets-with-missing-text-attribtute.json b/pxf/pxf-json/src/test/resources/tweets-with-missing-text-attribtute.json new file mode 100644 index 0000000..0353269 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/tweets-with-missing-text-attribtute.json @@ -0,0 +1,18 @@ +{ + "created_at":"Fri Jun 07 22:45:02 +0000 2013", + "id":343136547115253761, + "id_str":"343136547115253761", + "source":"<a href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\">Twitter for iPhone<\/a>", + "user":{ + "id":26643566, + "name":"SpreadButter", + "screen_name":"SpreadButter", + "location":"Austin, Texas" + }, + "entities":{ + "hashtags":[ + "tweetCongress", + "IRS" + ] + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/tweets.tar.gz ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/tweets.tar.gz b/pxf/pxf-json/src/test/resources/tweets.tar.gz new file mode 100644 index 0000000..fd5c9a6 Binary files /dev/null and b/pxf/pxf-json/src/test/resources/tweets.tar.gz differ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-json/src/test/resources/variable-size-objects.json ---------------------------------------------------------------------- diff --git a/pxf/pxf-json/src/test/resources/variable-size-objects.json b/pxf/pxf-json/src/test/resources/variable-size-objects.json new file mode 100644 index 0000000..58aa6c5 --- /dev/null +++ b/pxf/pxf-json/src/test/resources/variable-size-objects.json @@ -0,0 +1,3 @@ +{"key666":"small object1"} +{"key666":"large object2 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"} +{"key666":"small object3"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath b/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath index bc00ec7..c79f367 100644 --- a/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath +++ b/pxf/pxf-service/src/main/resources/pxf-privatehdp.classpath @@ -49,3 +49,4 @@ /usr/lib/pxf/pxf-hbase-*[0-9].jar /usr/lib/pxf/pxf-hdfs-*[0-9].jar /usr/lib/pxf/pxf-hive-*[0-9].jar +/usr/lib/pxf/pxf-json-*[0-9].jar http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath b/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath index 81cfe05..98d23a0 100644 --- a/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath +++ b/pxf/pxf-service/src/main/resources/pxf-privatephd.classpath @@ -49,3 +49,4 @@ /usr/lib/pxf/pxf-hbase-*[0-9].jar /usr/lib/pxf/pxf-hdfs-*[0-9].jar /usr/lib/pxf/pxf-hive-*[0-9].jar +/usr/lib/pxf/pxf-json-*[0-9].jar \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml index d908935..d72df94 100644 --- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml +++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml @@ -130,4 +130,18 @@ under the License. <resolver>org.apache.hawq.pxf.plugins.gemfirexd.GemFireXDResolver</resolver> </plugins> </profile> + <profile> + <name>Json</name> + <description> + Access JSON data either as: + * one JSON record per line (default) + * or multiline JSON records with an IDENTIFIER parameter indicating a member name + used to determine the encapsulating json object to return + </description> + <plugins> + <fragmenter>org.apache.hawq.pxf.plugins.hdfs.HdfsDataFragmenter</fragmenter> + <accessor>org.apache.hawq.pxf.plugins.json.JsonAccessor</accessor> + <resolver>org.apache.hawq.pxf.plugins.json.JsonResolver</resolver> + </plugins> + </profile> </profiles> http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fd9c3686/pxf/settings.gradle ---------------------------------------------------------------------- diff --git a/pxf/settings.gradle b/pxf/settings.gradle index c1111de..718eed5 100644 --- a/pxf/settings.gradle +++ b/pxf/settings.gradle @@ -24,3 +24,4 @@ include 'pxf-service' include 'pxf-hdfs' include 'pxf-hive' include 'pxf-hbase' +include 'pxf-json'
