Added: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java?rev=782092&view=auto ============================================================================== --- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java (added) +++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java Fri Jun 5 18:31:45 2009 @@ -0,0 +1,450 @@ +package org.apache.avro.io; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.Stack; + +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.JsonParser; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class TestBlockingIO { + private static final String UTF_8 = "UTF-8"; + + private static class Tests { + private final JsonParser parser; + private final ValueReader input; + private final int depth; + public Tests(int bufferSize, int depth, String input) + throws JsonParseException, IOException { + + this.depth = depth; + byte[] in = input.getBytes("UTF-8"); + JsonFactory f = new JsonFactory(); + JsonParser p = f.createJsonParser( + new ByteArrayInputStream(input.getBytes("UTF-8"))); + + ByteArrayOutputStream os = new ByteArrayOutputStream(); + ValueWriter cos = new BlockingValueWriter(os, bufferSize); + serialize(cos, p, os); + cos.flush(); + + byte[] bb = os.toByteArray(); + // dump(bb); + this.input = new ValueReader(new ByteArrayInputStream(bb)); + this.parser = f.createJsonParser(new ByteArrayInputStream(in)); + } + + public void scan() + throws JsonParseException, UnsupportedEncodingException, IOException { + Stack<S> countStack = new Stack<S>(); + long count = 0; + while (parser.nextToken() != null) { + switch (parser.getCurrentToken()) { + case END_ARRAY: + Assert.assertEquals(0, count); + Assert.assertTrue(countStack.peek().isArray); + count = countStack.pop().count; + break; + case END_OBJECT: + Assert.assertEquals(0, count); + Assert.assertFalse(countStack.peek().isArray); + count = countStack.pop().count; + break; + case START_ARRAY: + countStack.push(new S(count, true)); + count = input.readArrayStart(); + continue; + case VALUE_STRING: + { + String s = parser.getText(); + int n = s.getBytes(UTF_8).length; + checkString(s, input, n); + break; + } + case FIELD_NAME: + { + String s = parser.getCurrentName(); + int n = s.getBytes(UTF_8).length; + checkString(s, input, n); + continue; + } + case START_OBJECT: + countStack.push(new S(count, false)); + count = input.readMapStart(); + if (count < 0) { + count = -count; + input.readLong(); // byte count + } + continue; + default: + throw new RuntimeException("Unsupported: " + parser.getCurrentToken()); + } + count--; + if (count == 0) { + count = countStack.peek().isArray ? input.arrayNext() : + input.mapNext(); + } + } + } + + public void skip(int skipLevel) throws + JsonParseException, UnsupportedEncodingException, IOException { + Stack<S> countStack = new Stack<S>(); + long count = 0; + while (parser.nextToken() != null) { + switch (parser.getCurrentToken()) { + case END_ARRAY: + // assertEquals(0, count); + Assert.assertTrue(countStack.peek().isArray); + count = countStack.pop().count; + break; + case END_OBJECT: + // assertEquals(0, count); + Assert.assertFalse(countStack.peek().isArray); + count = countStack.pop().count; + break; + case START_ARRAY: + if (countStack.size() == skipLevel) { + skipArray(parser, input, depth - skipLevel); + break; + } else { + countStack.push(new S(count, true)); + count = input.readArrayStart(); + continue; + } + case VALUE_STRING: + { + if (countStack.size() == skipLevel) { + input.skipBytes(); + } else { + String s = parser.getText(); + int n = s.getBytes(UTF_8).length; + checkString(s, input, n); + } + break; + } + case FIELD_NAME: + { + String s = parser.getCurrentName(); + int n = s.getBytes(UTF_8).length; + checkString(s, input, n); + continue; + } + case START_OBJECT: + if (countStack.size() == skipLevel) { + skipMap(parser, input, depth - skipLevel); + break; + } else { + countStack.push(new S(count, false)); + count = input.readMapStart(); + if (count < 0) { + count = -count; + input.readLong(); // byte count + } + continue; + } + default: + throw new RuntimeException("Unsupported: " + parser.getCurrentToken()); + } + count--; + if (count == 0) { + count = countStack.peek().isArray ? input.arrayNext() : + input.mapNext(); + } + } + } + } + + protected static void dump(byte[] bb) { + int col = 0; + for (byte b : bb) { + if (col % 16 == 0) { + System.out.println(); + } + col++; + System.out.print(Integer.toHexString(b & 0xff) + " "); + } + System.out.println(); + } + + private static class S { + public final long count; + public final boolean isArray; + + public S(long count, boolean isArray) { + this.count = count; + this.isArray = isArray; + } + } + + @Test(dataProvider="data") + public void testScan(int bufferSize, int depth, String input) + throws JsonParseException, IOException { + Tests t = new Tests(bufferSize, depth, input); + t.scan(); + } + + @Test(dataProvider="data") + public void testSkip_1(int bufferSize, int depth, String input) + throws JsonParseException, IOException { + testSkip(bufferSize, depth, input, 0); + } + + @Test(dataProvider="data") + public void testSkip_2(int bufferSize, int depth, String input) + throws JsonParseException, IOException { + testSkip(bufferSize, depth, input, 1); + } + + @Test(dataProvider="data") + public void testSkip_3(int bufferSize, int depth, String input) + throws JsonParseException, IOException { + testSkip(bufferSize, depth, input, 2); + } + + private void testSkip(int bufferSize, int depth, String input, + int skipLevel) + throws JsonParseException, IOException { + Tests t = new Tests(bufferSize, depth, input); + t.skip(skipLevel); + } + + private static void skipMap(JsonParser parser, ValueReader input, int depth) + throws IOException, JsonParseException { + for (long l = input.skipMap(); l != 0; l = input.skipMap()) { + for (long i = 0; i < l; i++) { + if (depth == 0) { + input.skipBytes(); + } else { + skipArray(parser, input, depth - 1); + } + } + } + parser.skipChildren(); + } + + private static void skipArray(JsonParser parser, ValueReader input, int depth) + throws IOException, JsonParseException { + for (long l = input.skipArray(); l != 0; l = input.skipArray()) { + for (long i = 0; i < l; i++) { + if (depth == 1) { + input.skipBytes(); + } else { + skipArray(parser, input, depth - 1); + } + } + } + parser.skipChildren(); + } + + private static void checkString(String s, ValueReader input, int n) + throws IOException, UnsupportedEncodingException { + ByteBuffer buf = input.readBytes(null); + Assert.assertEquals(n, buf.remaining()); + String s2 = new String(buf.array(), buf.position(), + buf.remaining(), UTF_8); + Assert.assertEquals(s, s2); + } + + private static void serialize(ValueWriter cos, JsonParser p, + ByteArrayOutputStream os) + throws JsonParseException, IOException { + boolean[] isArray = new boolean[100]; + int[] counts = new int[100]; + int stackTop = -1; + + while (p.nextToken() != null) { + switch (p.getCurrentToken()) { + case END_ARRAY: + Assert.assertTrue(isArray[stackTop]); + cos.writeArrayEnd(); + stackTop--; + break; + case END_OBJECT: + Assert.assertFalse(isArray[stackTop]); + cos.writeMapEnd(); + stackTop--; + break; + case START_ARRAY: + if (stackTop >= 0 && isArray[stackTop]) { + cos.setItemCount(1); + cos.startItem(); + counts[stackTop]++; + } + cos.writeArrayStart(); + isArray[++stackTop] = true; + counts[stackTop] = 0; + continue; + case VALUE_STRING: + if (stackTop >= 0 && isArray[stackTop]) { + cos.setItemCount(1); + cos.startItem(); + counts[stackTop]++; + } + byte[] bb = p.getText().getBytes(UTF_8); + cos.writeBytes(bb); + break; + case START_OBJECT: + if (stackTop >= 0 && isArray[stackTop]) { + cos.setItemCount(1); + cos.startItem(); + counts[stackTop]++; + } + cos.writeMapStart(); + isArray[++stackTop] = false; + counts[stackTop] = 0; + continue; + case FIELD_NAME: + cos.setItemCount(1); + cos.startItem(); + counts[stackTop]++; + cos.writeBytes(p.getCurrentName().getBytes(UTF_8)); + break; + default: + throw new RuntimeException("Unsupported: " + p.getCurrentToken()); + } + } + } + + @DataProvider + public static Object[][] data() { + return new Object[][] { + { 64, 0, "" }, + { 64, 0, jss(0, 'a') }, + { 64, 0, jss(3, 'a') }, + { 64, 0, jss(64, 'a') }, + { 64, 0, jss(65, 'a') }, + { 64, 0, jss(100, 'a') }, + { 64, 1, "[]" }, + { 64, 1, "[" + jss(0, 'a') + "]" }, + { 64, 1, "[" + jss(3, 'a') + "]" }, + { 64, 1, "[" + jss(61, 'a') + "]" }, + { 64, 1, "[" + jss(62, 'a') + "]" }, + { 64, 1, "[" + jss(64, 'a') + "]" }, + { 64, 1, "[" + jss(65, 'a') + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(0, '0') + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(10, '0') + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(63, '0') + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(64, '0') + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(65, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(0, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(10, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(51, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(52, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(54, '0') + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(55, '0') + "]" }, + + { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(0, '0') + + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(63, '0') + + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(64, '0') + + "]" }, + { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(65, '0') + + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(10, '0') + + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(23, '0') + + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(24, '0') + + "]" }, + { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(25, '0') + + "]" }, + { 64, 2, "[[]]"}, + { 64, 2, "[[" + jss(0, 'a') + "], []]" }, + { 64, 2, "[[" + jss(10, 'a') + "], []]" }, + { 64, 2, "[[" + jss(59, 'a') + "], []]" }, + { 64, 2, "[[" + jss(60, 'a') + "], []]" }, + { 64, 2, "[[" + jss(100, 'a') + "], []]" }, + { 64, 2, "[[" + jss(10, '0') + ", " + jss(53, 'a') + "], []]" }, + { 64, 2, "[[" + jss(10, '0') + ", " + jss(54, 'a') + "], []]" }, + { 64, 2, "[[" + jss(10, '0') + ", " + jss(55, 'a') + "], []]" }, + + { 64, 2, "[[], [" + jss(0, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(10, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(63, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(64, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(65, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(53, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(54, 'a') + "]]" }, + { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(55, 'a') + "]]" }, + + { 64, 2, "[[" + jss(10, '0') + "]]"}, + { 64, 2, "[[" + jss(62, '0') + "]]"}, + { 64, 2, "[[" + jss(63, '0') + "]]"}, + { 64, 2, "[[" + jss(64, '0') + "]]"}, + { 64, 2, "[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]"}, + { 64, 2, "[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]"}, + { 64, 2, "[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]"}, + { 64, 2, "[[" + jss(10, 'a') + ", " + jss(54, '0') + "]]"}, + { 64, 3, "[[[" + jss(10, '0') + "]]]"}, + { 64, 3, "[[[" + jss(62, '0') + "]]]"}, + { 64, 3, "[[[" + jss(63, '0') + "]]]"}, + { 64, 3, "[[[" + jss(64, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(10, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(52, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(53, '0') + "]]]"}, + { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"}, + + { 64, 2, "[[\"p\"], [\"mn\"]]"}, + { 64, 2, "[[\"pqr\"], [\"mn\"]]"}, + { 64, 2, "[[\"pqrstuvwxyz\"], [\"mn\"]]"}, + { 64, 2, "[[\"abc\", \"pqrstuvwxyz\"], [\"mn\"]]"}, + { 64, 2, "[[\"mn\"], [\"\"]]"}, + { 64, 2, "[[\"mn\"], \"abc\"]"}, + { 64, 2, "[[\"mn\"], \"abcdefghijk\"]"}, + { 64, 2, "[[\"mn\"], \"pqr\", \"abc\"]"}, + { 64, 2, "[[\"mn\"]]"}, + { 64, 2, "[[\"p\"], [\"mnopqrstuvwx\"]]"}, + { 64, 2, "[[\"pqr\"], [\"mnopqrstuvwx\"]]"}, + { 64, 2, "[[\"pqrstuvwxyz\"], [\"mnopqrstuvwx\"]]"}, + { 64, 2, "[[\"abc\"], \"pqrstuvwxyz\", [\"mnopqrstuvwx\"]]"}, + { 64, 2, "[[\"mnopqrstuvwx\"], [\"\"]]"}, + { 64, 2, "[[\"mnopqrstuvwx\"], [\"abc\"]]"}, + { 64, 2, "[[\"mnopqrstuvwx\"], [\"abcdefghijk\"]]"}, + { 64, 2, "[[\"mnopqrstuvwx\"], [\"pqr\", \"abc\"]]"}, + { 100, 2, "[[\"pqr\", \"mnopqrstuvwx\"]]"}, + { 100, 2, "[[\"pqr\", \"ab\", \"mnopqrstuvwx\"]]"}, + { 64, 2, "[[[\"pqr\"]], [[\"ab\"], [\"mnopqrstuvwx\"]]]"}, + + { 64, 1, "{}" }, + { 64, 1, "{\"n\": \"v\"}" }, + { 64, 1, "{\"n1\": \"v\", \"n2\": []}" }, + { 100, 1, "{\"n1\": \"v\", \"n2\": []}" }, + { 100, 1, "{\"n1\": \"v\", \"n2\": [\"abc\"]}" }, + }; + } + + /** + * Returns a new JSON String {...@code n} bytes long with + * consecutive characters starting with {...@code c}. + */ + private static String jss(final int n, char c) { + char[] cc = new char[n + 2]; + cc[0] = cc[n + 1] = '"'; + for (int i = 1; i < n + 1; i++) { + if (c == 'Z') { + c = 'a'; + } else if (c == 'z') { + c = '0'; + } else if (c == '9') { + c = 'A'; + } else { + c++; + } + cc[i] = c; + } + return new String(cc); + } +}
Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java?rev=782092&r1=782091&r2=782092&view=diff ============================================================================== --- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java (original) +++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java Fri Jun 5 18:31:45 2009 @@ -30,46 +30,51 @@ public class TestValueReader { /** Verify EOFException throw at EOF */ - @Test - public void testEOFHandling() throws IOException { - InputStream is = new ByteArrayInputStream(new byte[0]); - ValueReader vr = new ValueReader(is); + @Test(expectedExceptions=EOFException.class) + public void testEOF_boolean() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readBoolean(); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_int() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readInt(); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_long() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readLong(); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_float() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readFloat(); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_double() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readDouble(); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_bytes() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readBytes(null); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_string() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])). + readString(new Utf8("a")); + } + + @Test(expectedExceptions=EOFException.class) + public void testEOF_fixed() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])). + readFixed(new byte[1]); + } - try { - vr.readBoolean(); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readBuffer(null); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readBytes(new byte[1]); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readBytes(new byte[1], 0, 1); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readDouble(); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readFloat(); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readInt(); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readLong(); - fail(); - } catch (EOFException e) { /* this is good */ } - try { - vr.readUtf8(new Utf8("a")); - fail(); - } catch (EOFException e) { /* this is good */ } + @Test(expectedExceptions=EOFException.class) + public void testEOF_enum() throws IOException { + new ValueReader(new ByteArrayInputStream(new byte[0])).readEnum(); } + }
