[OLINGO-568] Added implicit AND support
Project: http://git-wip-us.apache.org/repos/asf/olingo-odata4/repo Commit: http://git-wip-us.apache.org/repos/asf/olingo-odata4/commit/bbdd0d75 Tree: http://git-wip-us.apache.org/repos/asf/olingo-odata4/tree/bbdd0d75 Diff: http://git-wip-us.apache.org/repos/asf/olingo-odata4/diff/bbdd0d75 Branch: refs/heads/master Commit: bbdd0d755ed43d61b61d5eba87aed6d2cab410c4 Parents: 762c924 Author: Michael Bolz <[email protected]> Authored: Mon Nov 9 12:02:02 2015 +0100 Committer: Michael Bolz <[email protected]> Committed: Mon Nov 9 12:02:02 2015 +0100 ---------------------------------------------------------------------- .../core/uri/parser/search/SearchTokenizer.java | 62 +++++++++++++++++--- .../uri/parser/search/SearchParserTest.java | 5 +- .../uri/parser/search/SearchTokenizerTest.java | 39 +++++++++--- 3 files changed, 89 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/bbdd0d75/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java index 951d641..e058a00 100644 --- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java @@ -39,6 +39,7 @@ import java.util.List; * </code> */ public class SearchTokenizer { + public static final char QUOTATION_MARK = '\"'; //RWS = 1*( SP / HTAB / "%20" / "%09" ) ; "required" whitespace //BWS = *( SP / HTAB / "%20" / "%09" ) ; "bad" whitespace @@ -170,6 +171,8 @@ public class SearchTokenizer { return new CloseState(); } else if(isEof(c)) { return finish(); + } else if(isWhitespace(c)) { + return new AndState(c); } else { return new SearchTermState().init(c); } @@ -189,7 +192,7 @@ public class SearchTokenizer { public State nextChar(char c) { if(c == 'n' || c == 'N') { return new NotState(c); - } else if (c == '\'') { + } else if (c == QUOTATION_MARK) { return new SearchPhraseState(c); } else if (isAllowedChar(c)) { return new SearchWordState(c); @@ -230,7 +233,7 @@ public class SearchTokenizer { private class SearchPhraseState extends LiteralState { public SearchPhraseState(char c) { super(Token.PHRASE, c); - if(c != '\'') { + if(c != QUOTATION_MARK) { forbidden(c); } } @@ -241,7 +244,7 @@ public class SearchTokenizer { return new SearchExpressionState().init(c); } else if (isAllowedPhrase(c)) { return allowed(c); - } else if (c == '\'') { + } else if (c == QUOTATION_MARK) { finish(); return allowed(c); } else if (isWhitespace(c)) { @@ -299,13 +302,29 @@ public class SearchTokenizer { return allowed(c); } else if(getLiteral().length() == 3 && isWhitespace(c)) { finish(); - return new RwsState(); + return new BeforeSearchExpressionRwsState(); } else { return new SearchWordState(this); } } } + private class ImplicitAndState extends LiteralState { + private State followingState; + public ImplicitAndState(char c) { + super(Token.AND); + finish(); + followingState = new SearchExpressionState().init(c); + } + public State nextState() { + return followingState; + } + @Override + public State nextChar(char c) { + return followingState.nextChar(c); + } + } + private class AndState extends LiteralState { public AndState(char c) { super(Token.AND, c); @@ -321,7 +340,7 @@ public class SearchTokenizer { return allowed(c); } else if(getLiteral().length() == 3 && isWhitespace(c)) { finish(); - return new RwsState(); + return new BeforeSearchExpressionRwsState(); } else { return new SearchWordState(this); } @@ -341,13 +360,28 @@ public class SearchTokenizer { return allowed(c); } else if(getLiteral().length() == 2 && isWhitespace(c)) { finish(); - return new RwsState(); + return new BeforeSearchExpressionRwsState(); } else { return new SearchWordState(this); } } } + // RWS [ 'AND' RWS ] searchExpr + private class BeforeSearchExpressionRwsState extends State { + public BeforeSearchExpressionRwsState() { + super(Token.RWS); + } + @Override + public State nextChar(char c) { + if (isWhitespace(c)) { + return allowed(c); + } else { + return new SearchExpressionState().init(c); + } + } + } + private class RwsState extends State { public RwsState() { @@ -362,7 +396,7 @@ public class SearchTokenizer { } else if (c == 'A' || c == 'a') { return new AndState(c); } else { - return new SearchExpressionState().init(c); + return new ImplicitAndState(c); } } } @@ -373,16 +407,26 @@ public class SearchTokenizer { State state = new SearchExpressionState(); List<SearchQueryToken> states = new ArrayList<SearchQueryToken>(); + State lastAddedToken = null; for (char aChar : chars) { State next = state.nextChar(aChar); - if (state.isFinished() && next != state) { + if (next instanceof ImplicitAndState) { + lastAddedToken = next; + states.add(next); + next = ((ImplicitAndState)next).nextState(); + } else if (state.isFinished() && state != lastAddedToken) { + lastAddedToken = state; states.add(state); } state = next; } - if(state.nextChar(State.EOF).isFinished()) { + final State lastState = state.nextChar(State.EOF); + if(lastState.isFinished()) { states.add(state); + if(state.getToken() != lastState.getToken()) { + states.add(lastState); + } } else { throw new IllegalStateException("State: " + state + " not finished and list is: " + states.toString()); } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/bbdd0d75/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java index 81147ba..0c51ba6 100644 --- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java @@ -18,6 +18,8 @@ */ package org.apache.olingo.server.core.uri.parser.search; +import org.apache.olingo.server.api.uri.queryoption.SearchOption; +import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression; import org.junit.Test; public class SearchParserTest { @@ -25,6 +27,7 @@ public class SearchParserTest { @Test public void basicParsing() { SearchParser parser = new SearchParser(); - parser.parse("ESAllPrim", "abc"); + SearchOption so = parser.parse("ESAllPrim", "abc"); + SearchExpression se = so.getSearchExpression(); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/bbdd0d75/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java index 24d782f..ceae8d8 100644 --- a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java @@ -87,24 +87,24 @@ public class SearchTokenizerTest { List<SearchQueryToken> result; // - result = tokenizer.tokenize("'abc'"); + result = tokenizer.tokenize("\"abc\""); Assert.assertNotNull(result); log(result.toString()); Assert.assertEquals(PHRASE, result.get(0).getToken()); // - result = tokenizer.tokenize("'9988 abs'"); + result = tokenizer.tokenize("\"9988 abs\""); Assert.assertNotNull(result); log(result.toString()); Assert.assertEquals(PHRASE, result.get(0).getToken()); - Assert.assertEquals("'9988 abs'", result.get(0).getLiteral()); + Assert.assertEquals("\"9988 abs\"", result.get(0).getLiteral()); // - result = tokenizer.tokenize("'99_88.'"); + result = tokenizer.tokenize("\"99_88.\""); Assert.assertNotNull(result); log(result.toString()); Assert.assertEquals(PHRASE, result.get(0).getToken()); - Assert.assertEquals("'99_88.'", result.get(0).getLiteral()); + Assert.assertEquals("\"99_88.\"", result.get(0).getLiteral()); } @Test @@ -142,6 +142,15 @@ public class SearchTokenizerTest { } @Test + public void parseImplicitAnd() { + SearchValidator.init("a b").addExpected(WORD, AND, WORD).validate(); + SearchValidator.init("a b OR c").addExpected(WORD, AND, WORD, OR, WORD).validate(); + SearchValidator.init("a bc OR c").addExpected(WORD, AND, WORD, OR, WORD).validate(); + SearchValidator.init("a bc c").addExpected(WORD, AND, WORD, AND, WORD).validate(); + SearchValidator.init("(a OR x) bc c").addExpected(OPEN, WORD, OR, WORD, CLOSE, AND, WORD, AND, WORD).validate(); + } + + @Test public void testParseAnd() throws Exception { SearchTokenizer tokenizer = new SearchTokenizer(); List<SearchQueryToken> result; @@ -153,6 +162,22 @@ public class SearchTokenizerTest { Assert.assertEquals(AND, result.get(1).getToken()); Assert.assertEquals(WORD, result.get(2).getToken()); + // no lower case allowed for AND + result = tokenizer.tokenize("abc and xyz"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + + // implicit AND + result = tokenizer.tokenize("abc xyz"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + result = tokenizer.tokenize("abc AND xyz AND 123"); Assert.assertNotNull(result); log(result.toString()); @@ -162,13 +187,13 @@ public class SearchTokenizerTest { Assert.assertEquals(AND, result.get(3).getToken()); Assert.assertEquals(WORD, result.get(4).getToken()); - result = tokenizer.tokenize("abc AND 'x-y_z' AND 123"); + result = tokenizer.tokenize("abc AND \"x-y_z\" AND 123"); Assert.assertNotNull(result); log(result.toString()); Assert.assertEquals(WORD, result.get(0).getToken()); Assert.assertEquals(AND, result.get(1).getToken()); Assert.assertEquals(PHRASE, result.get(2).getToken()); - Assert.assertEquals("'x-y_z'", result.get(2).getLiteral()); + Assert.assertEquals("\"x-y_z\"", result.get(2).getLiteral()); Assert.assertEquals(AND, result.get(3).getToken()); Assert.assertEquals(WORD, result.get(4).getToken()); }
