Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff ============================================================================== --- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java (original) +++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/LexerTest.java Mon Jun 13 06:56:49 2016 @@ -52,345 +52,341 @@ public class LexerTest { formatWithEscaping = CSVFormat.DEFAULT.withEscape('\\'); } - private Lexer getLexer(final String input, final CSVFormat format) { + private Lexer createLexer(final String input, final CSVFormat format) { return new Lexer(format, new ExtendedBufferedReader(new StringReader(input))); } @Test public void testSurroundingSpacesAreDeleted() throws IOException { final String code = "noSpaces, leadingSpaces,trailingSpaces , surroundingSpaces , ,,"; - final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces()); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); + try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "noSpaces")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingSpaces")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingSpaces")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingSpaces")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } @Test public void testSurroundingTabsAreDeleted() throws IOException { final String code = "noTabs,\tleadingTab,trailingTab\t,\tsurroundingTabs\t,\t\t,,"; - final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces()); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); + try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "noTabs")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "leadingTab")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "trailingTab")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "surroundingTabs")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } @Test public void testIgnoreEmptyLines() throws IOException { - final String code = - "first,line,\n"+ - "\n"+ - "\n"+ - "second,line\n"+ - "\n"+ - "\n"+ - "third line \n"+ - "\n"+ - "\n"+ - "last, line \n"+ - "\n"+ - "\n"+ - "\n"; + final String code = "first,line,\n" + "\n" + "\n" + "second,line\n" + "\n" + "\n" + "third line \n" + "\n" + + "\n" + "last, line \n" + "\n" + "\n" + "\n"; final CSVFormat format = CSVFormat.DEFAULT.withIgnoreEmptyLines(); - final Lexer parser = getLexer(code, format); - - assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "line")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line ")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "last")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, " line ")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "third line ")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "last")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, " line ")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } @Test public void testComments() throws IOException { - final String code = - "first,line,\n"+ - "second,line,tokenWith#no-comment\n"+ - "# comment line \n"+ - "third,line,#no-comment\n"+ - "# penultimate comment\n"+ - "# Final comment\n"; + final String code = "first,line,\n" + "second,line,tokenWith#no-comment\n" + "# comment line \n" + + "third,line,#no-comment\n" + "# penultimate comment\n" + "# Final comment\n"; final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#'); - final Lexer parser = getLexer(code, format); - - assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment")); - assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "third")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); - assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); - assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "first")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "second")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "tokenWith#no-comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "comment line")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "third")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "line")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); + assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } @Test public void testCommentsAndEmptyLines() throws IOException { - final String code = - "1,2,3,\n"+ // 1 - "\n"+ // 1b - "\n"+ // 1c - "a,b x,c#no-comment\n"+ // 2 - "#foo\n"+ // 3 - "\n"+ // 4 - "\n"+ // 4b - "d,e,#no-comment\n"+ // 5 - "\n"+ // 5b - "\n"+ // 5c - "# penultimate comment\n"+ // 6 - "\n"+ // 6b - "\n"+ // 6c - "# Final comment\n"; // 7 + final String code = "1,2,3,\n" + // 1 + "\n" + // 1b + "\n" + // 1c + "a,b x,c#no-comment\n" + // 2 + "#foo\n" + // 3 + "\n" + // 4 + "\n" + // 4b + "d,e,#no-comment\n" + // 5 + "\n" + // 5b + "\n" + // 5c + "# penultimate comment\n" + // 6 + "\n" + // 6b + "\n" + // 6c + "# Final comment\n"; // 7 final CSVFormat format = CSVFormat.DEFAULT.withCommentMarker('#').withIgnoreEmptyLines(false); assertFalse("Should not ignore empty lines", format.getIgnoreEmptyLines()); - final Lexer parser = getLexer(code, format); - - - assertThat(parser.nextToken(new Token()), matches(TOKEN, "1")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "2")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "3")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1 - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2 - assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3 - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4 - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b - assertThat(parser.nextToken(new Token()), matches(TOKEN, "d")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "e")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5 - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c - assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6 - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b - assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c - assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7 - assertThat(parser.nextToken(new Token()), matches(EOF, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); - + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "1")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "2")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "3")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1 + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1b + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 1c + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "b x")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "c#no-comment")); // 2 + assertThat(parser.nextToken(new Token()), matches(COMMENT, "foo")); // 3 + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4 + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 4b + assertThat(parser.nextToken(new Token()), matches(TOKEN, "d")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "e")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "#no-comment")); // 5 + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5b + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 5c + assertThat(parser.nextToken(new Token()), matches(COMMENT, "penultimate comment")); // 6 + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6b + assertThat(parser.nextToken(new Token()), matches(EORECORD, "")); // 6c + assertThat(parser.nextToken(new Token()), matches(COMMENT, "Final comment")); // 7 + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } // simple token with escaping not enabled @Test public void testBackslashWithoutEscaping() throws IOException { - /* file: a,\,,b - * \,, - */ + /* + * file: a,\,,b \,, + */ final String code = "a,\\,,b\\\n\\,,"; final CSVFormat format = CSVFormat.DEFAULT; assertFalse(format.isEscapeCharacterSet()); - final Lexer parser = getLexer(code, format); - - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - // an unquoted single backslash is not an escape char - assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\")); - // an unquoted single backslash is not an escape char - assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(EOF, "")); + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + // an unquoted single backslash is not an escape char + assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\")); + // an unquoted single backslash is not an escape char + assertThat(parser.nextToken(new Token()), matches(TOKEN, "\\")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(EOF, "")); + } } // simple token with escaping enabled @Test public void testBackslashWithEscaping() throws IOException { - /* file: a,\,,b - * \,, - */ + /* + * file: a,\,,b \,, + */ final String code = "a,\\,,b\\\\\n\\,,\\\nc,d\\\r\ne"; final CSVFormat format = formatWithEscaping.withIgnoreEmptyLines(false); assertTrue(format.isEscapeCharacterSet()); - final Lexer parser = getLexer(code, format); - - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, ",")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, ",")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r")); - assertThat(parser.nextToken(new Token()), matches(EOF, "e")); + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, ",")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b\\")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, ",")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "\nc")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "d\r")); + assertThat(parser.nextToken(new Token()), matches(EOF, "e")); + } } // encapsulator tokenizer (single line) @Test public void testNextToken4() throws IOException { - /* file: a,"foo",b - * a, " foo",b - * a,"foo " ,b // whitespace after closing encapsulator - * a, " foo " ,b - */ + /* + * file: a,"foo",b a, " foo",b a,"foo " ,b // whitespace after closing encapsulator a, " foo " ,b + */ final String code = "a,\"foo\",b\na, \" foo\",b\na,\"foo \" ,b\na, \" foo \" ,b"; - final Lexer parser = getLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces()); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo ")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo ")); -// assertTokenEquals(EORECORD, "b", parser.nextToken(new Token())); - assertThat(parser.nextToken(new Token()), matches(EOF, "b")); + try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo ")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, " foo ")); + // assertTokenEquals(EORECORD, "b", parser.nextToken(new Token())); + assertThat(parser.nextToken(new Token()), matches(EOF, "b")); + } } // encapsulator tokenizer (multi line, delimiter in string) @Test public void testNextToken5() throws IOException { final String code = "a,\"foo\n\",b\n\"foo\n baar ,,,\"\n\"\n\t \n\""; - final Lexer parser = getLexer(code, CSVFormat.DEFAULT); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,")); - assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n")); - + try (final Lexer parser = createLexer(code, CSVFormat.DEFAULT)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "foo\n")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "foo\n baar ,,,")); + assertThat(parser.nextToken(new Token()), matches(EOF, "\n\t \n")); + } } // change delimiters, comment, encapsulater @Test public void testNextToken6() throws IOException { - /* file: a;'b and \' more - * ' - * !comment;;;; - * ;; - */ + /* + * file: a;'b and \' more ' !comment;;;; ;; + */ final String code = "a;'b and '' more\n'\n!comment;;;;\n;;"; final CSVFormat format = CSVFormat.DEFAULT.withQuote('\'').withCommentMarker('!').withDelimiter(';'); - final Lexer parser = getLexer(code, format); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); - assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n")); + try (final Lexer parser = createLexer(code, format)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "a")); + assertThat(parser.nextToken(new Token()), matches(EORECORD, "b and ' more\n")); + } } // From CSV-1 @Test public void testDelimiterIsWhitespace() throws IOException { final String code = "one\ttwo\t\tfour \t five\t six"; - final Lexer parser = getLexer(code, CSVFormat.TDF); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "one")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "two")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "four")); - assertThat(parser.nextToken(new Token()), matches(TOKEN, "five")); - assertThat(parser.nextToken(new Token()), matches(EOF, "six")); + try (final Lexer parser = createLexer(code, CSVFormat.TDF)) { + assertThat(parser.nextToken(new Token()), matches(TOKEN, "one")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "two")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "four")); + assertThat(parser.nextToken(new Token()), matches(TOKEN, "five")); + assertThat(parser.nextToken(new Token()), matches(EOF, "six")); + } } @Test public void testEscapedCR() throws Exception { - final Lexer lexer = getLexer("character\\" + CR + "Escaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + try (final Lexer lexer = createLexer("character\\" + CR + "Escaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + } } @Test public void testCR() throws Exception { - final Lexer lexer = getLexer("character" + CR + "NotEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character")); - assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped")); + try (final Lexer lexer = createLexer("character" + CR + "NotEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character")); + assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped")); + } } @Test public void testEscapedLF() throws Exception { - final Lexer lexer = getLexer("character\\" + LF + "Escaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped")); + try (final Lexer lexer = createLexer("character\\" + LF + "Escaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + LF + "Escaped")); + } } @Test public void testLF() throws Exception { - final Lexer lexer = getLexer("character" + LF + "NotEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character")); - assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped")); + try (final Lexer lexer = createLexer("character" + LF + "NotEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character")); + assertThat(lexer.nextToken(new Token()), hasContent("NotEscaped")); + } } @Test // TODO is this correct? Do we expect <esc>TAB to be unescaped? public void testEscapedTab() throws Exception { - final Lexer lexer = getLexer("character\\" + TAB + "Escaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped")); + try (final Lexer lexer = createLexer("character\\" + TAB + "Escaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "Escaped")); + } + } @Test public void testTab() throws Exception { - final Lexer lexer = getLexer("character" + TAB + "NotEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped")); + try (final Lexer lexer = createLexer("character" + TAB + "NotEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + TAB + "NotEscaped")); + } } @Test // TODO is this correct? Do we expect <esc>BACKSPACE to be unescaped? public void testEscapedBackspace() throws Exception { - final Lexer lexer = getLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped")); + try (final Lexer lexer = createLexer("character\\" + BACKSPACE + "Escaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "Escaped")); + } } @Test public void testBackspace() throws Exception { - final Lexer lexer = getLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped")); + try (final Lexer lexer = createLexer("character" + BACKSPACE + "NotEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + BACKSPACE + "NotEscaped")); + } } @Test // TODO is this correct? Do we expect <esc>FF to be unescaped? public void testEscapedFF() throws Exception { - final Lexer lexer = getLexer("character\\" + FF + "Escaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped")); + try (final Lexer lexer = createLexer("character\\" + FF + "Escaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "Escaped")); + } } @Test public void testFF() throws Exception { - final Lexer lexer = getLexer("character" + FF + "NotEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped")); + try (final Lexer lexer = createLexer("character" + FF + "NotEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + FF + "NotEscaped")); + } } @Test public void testEscapedMySqlNullValue() throws Exception { // MySQL uses \N to symbolize null values. We have to restore this - final Lexer lexer = getLexer("character\\NEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped")); + try (final Lexer lexer = createLexer("character\\NEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character\\NEscaped")); + } } @Test public void testEscapedCharacter() throws Exception { - final Lexer lexer = getLexer("character\\aEscaped", formatWithEscaping); - assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped")); + try (final Lexer lexer = createLexer("character\\aEscaped", formatWithEscaping)) { + assertThat(lexer.nextToken(new Token()), hasContent("character\\aEscaped")); + } } @Test public void testEscapedControlCharacter() throws Exception { // we are explicitly using an escape different from \ here - final Lexer lexer = getLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!')); - assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + try (final Lexer lexer = createLexer("character!rEscaped", CSVFormat.DEFAULT.withEscape('!'))) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + } } @Test public void testEscapedControlCharacter2() throws Exception { - final Lexer lexer = getLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\')); - assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + try (final Lexer lexer = createLexer("character\\rEscaped", CSVFormat.DEFAULT.withEscape('\\'))) { + assertThat(lexer.nextToken(new Token()), hasContent("character" + CR + "Escaped")); + } } @Test(expected = IOException.class) public void testEscapingAtEOF() throws Exception { final String code = "escaping at EOF is evil\\"; - final Lexer lexer = getLexer(code, formatWithEscaping); - - lexer.nextToken(new Token()); + try (final Lexer lexer = createLexer(code, formatWithEscaping)) { + lexer.nextToken(new Token()); + } } }
Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff ============================================================================== --- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java (original) +++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/PerformanceTest.java Mon Jun 13 06:56:49 2016 @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.zip.GZIPInputStream; import org.apache.commons.io.IOUtils; @@ -73,11 +74,11 @@ public class PerformanceTest { System.out.println(String.format("Found test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length())); } else { System.out.println("Decompressing test fixture " + BIG_FILE + "..."); - final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz")); - final OutputStream output = new FileOutputStream(BIG_FILE); - IOUtils.copy(input, output); - input.close(); - output.close(); + try (final InputStream input = new GZIPInputStream( + new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz")); + final OutputStream output = new FileOutputStream(BIG_FILE)) { + IOUtils.copy(input, output); + } System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length())); } final int argc = args.length; @@ -121,7 +122,7 @@ public class PerformanceTest { } } - private static BufferedReader getReader() throws IOException { + private static BufferedReader createReader() throws IOException { return new BufferedReader(new FileReader(BIG_FILE)); } @@ -155,15 +156,17 @@ public class PerformanceTest { } private static void testReadBigFile(final boolean split) throws Exception { - for (int i = 0; i < max; i++) { - final BufferedReader in = getReader(); - final long t0 = System.currentTimeMillis(); - final Stats s = readAll(in, split); - in.close(); - show(split?"file+split":"file", s, t0); - } - show(); - } + for (int i = 0; i < max; i++) { + final long startMillis; + final Stats stats; + try (final BufferedReader in = createReader()) { + startMillis = System.currentTimeMillis(); + stats = readAll(in, split); + } + show(split ? "file+split" : "file", stats, startMillis); + } + show(); + } private static Stats readAll(final BufferedReader in, final boolean split) throws IOException { int count = 0; @@ -176,55 +179,58 @@ public class PerformanceTest { return new Stats(count, fields); } - private static void testExtendedBuffer(final boolean makeString) throws Exception { - for (int i = 0; i < max; i++) { - final ExtendedBufferedReader in = new ExtendedBufferedReader(getReader()); - final long t0 = System.currentTimeMillis(); - int read; - int fields = 0; - int lines = 0; - if (makeString) { - StringBuilder sb = new StringBuilder(); - while((read=in.read()) != -1) { - sb.append((char)read); - if (read == ',') { // count delimiters - sb.toString(); - sb = new StringBuilder(); - fields++; - } else if (read == '\n') { - sb.toString(); - sb = new StringBuilder(); - lines++; - } - } - } else { - while((read=in.read()) != -1) { - if (read == ',') { // count delimiters - fields++; - } else if (read == '\n') { - lines++; - } - } - } - fields += lines; // EOL is a delimiter too - in.close(); - show("Extended"+(makeString?" toString":""), new Stats(lines, fields), t0); - } - show(); - } + private static void testExtendedBuffer(final boolean makeString) throws Exception { + for (int i = 0; i < max; i++) { + int fields = 0; + int lines = 0; + final long startMillis; + try (final ExtendedBufferedReader in = new ExtendedBufferedReader(createReader())) { + startMillis = System.currentTimeMillis(); + int read; + if (makeString) { + StringBuilder sb = new StringBuilder(); + while ((read = in.read()) != -1) { + sb.append((char) read); + if (read == ',') { // count delimiters + sb.toString(); + sb = new StringBuilder(); + fields++; + } else if (read == '\n') { + sb.toString(); + sb = new StringBuilder(); + lines++; + } + } + } else { + while ((read = in.read()) != -1) { + if (read == ',') { // count delimiters + fields++; + } else if (read == '\n') { + lines++; + } + } + } + fields += lines; // EOL is a delimiter too + } + show("Extended" + (makeString ? " toString" : ""), new Stats(lines, fields), startMillis); + } + show(); + } - private static void testParseCommonsCSV() throws Exception { - for (int i = 0; i < max; i++) { - final BufferedReader reader = getReader(); - final CSVParser parser = new CSVParser(reader, format); - final long t0 = System.currentTimeMillis(); - final Stats s = iterate(parser); - reader.close(); - show("CSV", s, t0); - parser.close(); - } - show(); - } + private static void testParseCommonsCSV() throws Exception { + for (int i = 0; i < max; i++) { + final long startMillis; + final Stats stats; + try (final BufferedReader reader = createReader()) { + try (final CSVParser parser = new CSVParser(reader, format)) { + startMillis = System.currentTimeMillis(); + stats = iterate(parser); + } + show("CSV", stats, startMillis); + } + } + show(); + } private static Constructor<Lexer> getLexerCtor(final String clazz) throws Exception { @@ -233,53 +239,59 @@ public class PerformanceTest { return lexer.getConstructor(new Class<?>[]{CSVFormat.class, ExtendedBufferedReader.class}); } - private static void testCSVLexer(final boolean newToken, final String test) throws Exception { - Token token = new Token(); - String dynamic = ""; - for (int i = 0; i < max; i++) { - final ExtendedBufferedReader input = new ExtendedBufferedReader(getReader()); - Lexer lexer = null; - if (test.startsWith("CSVLexer")) { - dynamic="!"; - lexer = getLexerCtor(test).newInstance(new Object[]{format, input}); - } else { - lexer = new Lexer(format, input); - } - int count = 0; - int fields = 0; - final long t0 = System.currentTimeMillis(); - do { - if (newToken) { - token = new Token(); - } else { - token.reset(); - } - lexer.nextToken(token); - switch(token.type) { - case EOF: - break; - case EORECORD: - fields++; - count++; - break; - case INVALID: - throw new IOException("invalid parse sequence <"+token.content.toString()+">"); - case TOKEN: - fields++; - break; - case COMMENT: // not really expecting these - break; - default: - throw new IllegalStateException("Unexpected Token type: " + token.type); - } - - } while (!token.type.equals(Token.Type.EOF)); - final Stats s = new Stats(count, fields); - input.close(); - show(lexer.getClass().getSimpleName()+dynamic+" "+(newToken ? "new" : "reset"), s, t0); - } - show(); - } + private static void testCSVLexer(final boolean newToken, final String test) throws Exception { + Token token = new Token(); + String dynamic = ""; + for (int i = 0; i < max; i++) { + final String simpleName; + final Stats stats; + final long startMillis; + try (final ExtendedBufferedReader input = new ExtendedBufferedReader(createReader()); + Lexer lexer = createTestCSVLexer(test, input)) { + if (test.startsWith("CSVLexer")) { + dynamic = "!"; + } + simpleName = lexer.getClass().getSimpleName(); + int count = 0; + int fields = 0; + startMillis = System.currentTimeMillis(); + do { + if (newToken) { + token = new Token(); + } else { + token.reset(); + } + lexer.nextToken(token); + switch (token.type) { + case EOF: + break; + case EORECORD: + fields++; + count++; + break; + case INVALID: + throw new IOException("invalid parse sequence <" + token.content.toString() + ">"); + case TOKEN: + fields++; + break; + case COMMENT: // not really expecting these + break; + default: + throw new IllegalStateException("Unexpected Token type: " + token.type); + } + } while (!token.type.equals(Token.Type.EOF)); + stats = new Stats(count, fields); + } + show(simpleName + dynamic + " " + (newToken ? "new" : "reset"), stats, startMillis); + } + show(); + } + + private static Lexer createTestCSVLexer(final String test, final ExtendedBufferedReader input) + throws InstantiationException, IllegalAccessException, InvocationTargetException, Exception { + return test.startsWith("CSVLexer") ? getLexerCtor(test) + .newInstance(new Object[] { format, input }) : new Lexer(format, input); + } private static Stats iterate(final Iterable<CSVRecord> it) { int count = 0; Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java?rev=1748094&r1=1748093&r2=1748094&view=diff ============================================================================== --- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java (original) +++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv164Test.java Mon Jun 13 06:56:49 2016 @@ -29,12 +29,13 @@ public class JiraCsv164Test { @Test public void testJiraCsv154_withCommentMarker() throws IOException { final String comment = "This is a header comment"; - final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#').withHeaderComments(comment); + final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withCommentMarker('#') + .withHeaderComments(comment); final StringBuilder out = new StringBuilder(); - final CSVPrinter printer = format.print(out); - printer.print("A"); - printer.print("B"); - printer.close(); + try (final CSVPrinter printer = format.print(out)) { + printer.print("A"); + printer.print("B"); + } final String s = out.toString(); assertTrue(s, s.contains(comment)); } @@ -42,12 +43,13 @@ public class JiraCsv164Test { @Test public void testJiraCsv154_withHeaderComments() throws IOException { final String comment = "This is a header comment"; - final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment).withCommentMarker('#'); + final CSVFormat format = CSVFormat.EXCEL.withHeader("H1", "H2").withHeaderComments(comment) + .withCommentMarker('#'); final StringBuilder out = new StringBuilder(); - final CSVPrinter printer = format.print(out); - printer.print("A"); - printer.print("B"); - printer.close(); + try (final CSVPrinter printer = format.print(out)) { + printer.print("A"); + printer.print("B"); + } final String s = out.toString(); assertTrue(s, s.contains(comment)); } Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java?rev=1748094&r1=1748093&r2=1748094&view=diff ============================================================================== --- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java (original) +++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/bugs/JiraCsv167Test.java Mon Jun 13 06:56:49 2016 @@ -33,23 +33,23 @@ public class JiraCsv167Test { @Test public void parse() throws IOException { - final BufferedReader br = new BufferedReader(getTestInput()); - String s = null; int totcomment = 0; int totrecs = 0; - boolean lastWasComment = false; - while((s=br.readLine()) != null) { - if (s.startsWith("#")) { - if (!lastWasComment) { // comments are merged - totcomment++; + try (final BufferedReader br = new BufferedReader(getTestInput())) { + String s = null; + boolean lastWasComment = false; + while ((s = br.readLine()) != null) { + if (s.startsWith("#")) { + if (!lastWasComment) { // comments are merged + totcomment++; + } + lastWasComment = true; + } else { + totrecs++; + lastWasComment = false; } - lastWasComment = true; - } else { - totrecs++; - lastWasComment = false; } } - br.close(); CSVFormat format = CSVFormat.DEFAULT; // format = format.withAllowMissingColumnNames(false); @@ -66,13 +66,14 @@ public class JiraCsv167Test { format = format.withRecordSeparator('\n'); format = format.withSkipHeaderRecord(false); // - final CSVParser parser = format.parse(getTestInput()); int comments = 0; int records = 0; - for (final CSVRecord csvRecord : parser) { - records++; - if (csvRecord.hasComment()) { - comments++; + try (final CSVParser parser = format.parse(getTestInput())) { + for (final CSVRecord csvRecord : parser) { + records++; + if (csvRecord.hasComment()) { + comments++; + } } } // Comment lines are concatenated, in this example 4 lines become 2 comments. Modified: commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java URL: http://svn.apache.org/viewvc/commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java?rev=1748094&r1=1748093&r2=1748094&view=diff ============================================================================== --- commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java (original) +++ commons/proper/csv/trunk/src/test/java/org/apache/commons/csv/perf/PerformanceTest.java Mon Jun 13 06:56:49 2016 @@ -56,15 +56,15 @@ public class PerformanceTest { return; } System.out.println("Decompressing test fixture " + BIG_FILE + "..."); - final InputStream input = new GZIPInputStream(new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz")); - final OutputStream output = new FileOutputStream(BIG_FILE); - IOUtils.copy(input, output); - System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length())); - input.close(); - output.close(); + try (final InputStream input = new GZIPInputStream( + new FileInputStream("src/test/resources/perf/worldcitiespop.txt.gz")); + final OutputStream output = new FileOutputStream(BIG_FILE)) { + IOUtils.copy(input, output); + System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length())); + } } - private BufferedReader getBufferedReader() throws IOException { + private BufferedReader createBufferedReader() throws IOException { return new BufferedReader(new FileReader(BIG_FILE)); } @@ -96,7 +96,7 @@ public class PerformanceTest { public long testParseBigFile(final boolean traverseColumns) throws Exception { final long startMillis = System.currentTimeMillis(); - final long count = this.parse(this.getBufferedReader(), traverseColumns); + final long count = this.parse(this.createBufferedReader(), traverseColumns); final long totalMillis = System.currentTimeMillis() - startMillis; this.println(String.format("File parsed in %,d milliseconds with Commons CSV: %,d lines.", totalMillis, count)); return totalMillis; @@ -115,13 +115,12 @@ public class PerformanceTest { public void testReadBigFile() throws Exception { long bestTime = Long.MAX_VALUE; for (int i = 0; i < this.max; i++) { - final BufferedReader in = this.getBufferedReader(); - final long startMillis = System.currentTimeMillis(); - long count = 0; - try { + final long startMillis; + long count; + try (final BufferedReader in = this.createBufferedReader()) { + startMillis = System.currentTimeMillis(); + count = 0; count = this.readAll(in); - } finally { - in.close(); } final long totalMillis = System.currentTimeMillis() - startMillis; bestTime = Math.min(totalMillis, bestTime);
