Repository: olingo-odata4 Updated Branches: refs/heads/master be3b10a24 -> ae1b2754b
[OLINGO-568] SearchTokenizer for SearchParser Project: http://git-wip-us.apache.org/repos/asf/olingo-odata4/repo Commit: http://git-wip-us.apache.org/repos/asf/olingo-odata4/commit/452ebcbd Tree: http://git-wip-us.apache.org/repos/asf/olingo-odata4/tree/452ebcbd Diff: http://git-wip-us.apache.org/repos/asf/olingo-odata4/diff/452ebcbd Branch: refs/heads/master Commit: 452ebcbdd80d358b71e1e1eb2f8a9830e7aaa59f Parents: ac828a3 Author: Michael Bolz <[email protected]> Authored: Fri Nov 6 15:42:32 2015 +0100 Committer: Michael Bolz <[email protected]> Committed: Fri Nov 6 15:43:36 2015 +0100 ---------------------------------------------------------------------- .../olingo/server/core/uri/parser/Parser.java | 21 +- .../core/uri/parser/search/SearchParser.java | 33 ++ .../uri/parser/search/SearchQueryToken.java | 26 ++ .../core/uri/parser/search/SearchTokenizer.java | 396 +++++++++++++++++++ .../core/uri/queryoption/SearchOptionImpl.java | 7 +- .../uri/parser/search/SearchParserTest.java | 30 ++ .../uri/parser/search/SearchTokenizerTest.java | 388 ++++++++++++++++++ .../core/uri/antlr/TestFullResourcePath.java | 17 +- 8 files changed, 902 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/Parser.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/Parser.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/Parser.java index 8732341..82094cf 100644 --- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/Parser.java +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/Parser.java @@ -55,6 +55,7 @@ import org.apache.olingo.server.core.uri.antlr.UriParserParser.MetadataEOFContex import org.apache.olingo.server.core.uri.antlr.UriParserParser.OrderByEOFContext; import org.apache.olingo.server.core.uri.antlr.UriParserParser.PathSegmentEOFContext; import org.apache.olingo.server.core.uri.antlr.UriParserParser.SelectEOFContext; +import org.apache.olingo.server.core.uri.parser.search.SearchParser; import org.apache.olingo.server.core.uri.queryoption.CountOptionImpl; import org.apache.olingo.server.core.uri.queryoption.CustomQueryOptionImpl; import org.apache.olingo.server.core.uri.queryoption.ExpandOptionImpl; @@ -77,7 +78,7 @@ public class Parser { int logLevel = 0; private enum ParserEntryRules { - All, Batch, CrossJoin, Entity, ExpandItems, FilterExpression, Metadata, PathSegment, Orderby, Select + All, Batch, CrossJoin, Entity, ExpandItems, FilterExpression, Metadata, PathSegment, Orderby, Select, Search } public Parser setLogLevel(final int logLevel) { @@ -215,8 +216,8 @@ public class Parser { systemOption = (OrderByOptionImpl) uriParseTreeVisitor.visitOrderByEOF(ctxOrderByExpression); } else if (option.name.equals(SystemQueryOptionKind.SEARCH.toString())) { - throw new UriParserSemanticException("System query option '$search' not implemented!", - UriParserSemanticException.MessageKeys.NOT_IMPLEMENTED, "System query option '$search"); + SearchParser searchParser = new SearchParser(); + systemOption = searchParser.parse(path, option.value); } else if (option.name.equals(SystemQueryOptionKind.SELECT.toString())) { SelectEOFContext ctxSelectEOF = (SelectEOFContext) parseRule(option.value, ParserEntryRules.Select); @@ -285,15 +286,15 @@ public class Parser { parameter.setAlias(option.name); parameter.setExpression(expression); parameter.setText(NULL.equals(option.value) ? null : option.value); - + if(context.contextUriInfo.getAlias(option.name) == null) { context.contextUriInfo.addAlias(option.name, parameter); } else { - throw new UriParserSyntaxException("Alias already specified! Name: " + option.name, + throw new UriParserSyntaxException("Alias already specified! Name: " + option.name, UriParserSyntaxException.MessageKeys.DUPLICATED_ALIAS, option.name); } } - + final CustomQueryOptionImpl customOption = new CustomQueryOptionImpl(); customOption.setName(option.name); customOption.setText(option.value); @@ -388,6 +389,9 @@ public class Parser { case Select: ret = parser.selectEOF(); break; + case Search: + ret = parser.searchInline(); + break; default: break; @@ -445,6 +449,9 @@ public class Parser { case Select: ret = parser.selectEOF(); break; + case Search: + ret = parser.searchInline(); + break; default: break; } @@ -503,7 +510,7 @@ public class Parser { } else { out.append(index); } - out.append(nL); + out.append(nL); } out.append(']'); System.out.println("tokens: " + out.toString()); http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java new file mode 100644 index 0000000..d508932 --- /dev/null +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchParser.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.olingo.server.core.uri.parser.search; + +import org.apache.olingo.server.api.uri.queryoption.SearchOption; +import org.apache.olingo.server.core.uri.queryoption.SearchOptionImpl; + +import java.util.List; + +public class SearchParser { + + public SearchOption parse(String path, String value) { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> tokens = tokenizer.tokenize(value); + return new SearchOptionImpl(); + } +} http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchQueryToken.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchQueryToken.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchQueryToken.java new file mode 100644 index 0000000..a08c1a2 --- /dev/null +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchQueryToken.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.olingo.server.core.uri.parser.search; + +public interface SearchQueryToken { + enum Token {OPEN, BWS, RWS, TERM, SEARCH_EXPRESSION, NOT, AND, OR, WORD, PHRASE, CLOSE} + + Token getToken(); + String getLiteral(); +} http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java new file mode 100644 index 0000000..7d2b559 --- /dev/null +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizer.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.olingo.server.core.uri.parser.search; + +import java.util.ArrayList; +import java.util.List; + +/** + * <code> + * searchExpr = ( OPEN BWS searchExpr BWS CLOSE + * / searchTerm + * ) [ searchOrExpr + * / searchAndExpr + * ] + * + * searchOrExpr = RWS 'OR' RWS searchExpr + * searchAndExpr = RWS [ 'AND' RWS ] searchExpr + * + * searchTerm = [ 'NOT' RWS ] ( searchPhrase / searchWord ) + * searchPhrase = quotation-mark 1*qchar-no-AMP-DQUOTE quotation-mark + * searchWord = 1*ALPHA ; Actually: any character from the Unicode categories L or Nl, + * ; but not the words AND, OR, and NOT + * </code> + */ +public class SearchTokenizer { + //RWS = 1*( SP / HTAB / "%20" / "%09" ) ; "required" whitespace + //BWS = *( SP / HTAB / "%20" / "%09" ) ; "bad" whitespace + + + private static abstract class State implements SearchQueryToken { + private Token token = null; + private boolean finished = false; + private final StringBuilder literal; + + public static final char EOF = 0x03; + + public State(Token t) { + token = t; + literal = new StringBuilder(); + } + public State(Token t, char c) { + this(t); + init(c); + } + public State(Token t, State consumeState) { + token = t; + literal = new StringBuilder(consumeState.getLiteral()); + } + + protected abstract State nextChar(char c); + + public State next(char c) { + return nextChar(c); + } + + public State init(char c) { + if(isFinished()) { + throw new IllegalStateException(toString() + " is already finished."); + } + literal.append(c); + return this; + } + + public State allowed(char c) { + literal.append(c); + return this; + } + + public State forbidden(char c) { + throw new IllegalStateException(this.getClass().getName() + "->" + c); + } + + public State finish() { + this.finished = true; + return this; + } + + public boolean isFinished() { + return finished; + } + + public Token getToken() { + return token; + } + + static boolean isAllowedChar(final char character) { + // TODO mibo: add missing allowed characters + return 'A' <= character && character <= 'Z' // case A..Z + || 'a' <= character && character <= 'z' // case a..z + || '0' <= character && character <= '9'; // case 0..9 + } + + /** + * qchar-no-AMP-DQUOTE = qchar-unescaped / escape ( escape / quotation-mark ) + * qchar-unescaped = unreserved / pct-encoded-unescaped / other-delims / ":" / "@" / "/" / "?" / "$" / "'" / "=" + * unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + * @param character which is checked + * @return true if character is allowed for a phrase + */ + static boolean isAllowedPhrase(final char character) { + // FIXME mibo: check missing and '\'' + return isAllowedChar(character) + || character == '-' + || character == '.' + || character == '_' + || character == '~' + || character == ':' + || character == '@' + || character == '/' + || character == '$' + || character == '='; + } + + static boolean isEof(final char character) { + return character == EOF; + } + + static boolean isWhitespace(final char character) { + //( SP / HTAB / "%20" / "%09" ) + // TODO mibo: add missing whitespaces + return character == ' ' || character == '\t'; + } + + @Override + public String getLiteral() { + return literal.toString(); + } + + @Override + public String toString() { + return this.getToken().toString() + "=>{" + literal.toString() + "}"; + } + } + + private class SearchExpressionState extends State { + public SearchExpressionState() { + super(Token.SEARCH_EXPRESSION); + } + @Override + public State nextChar(char c) { + if (c == '(') { + return new OpenState(c); + } else if (isWhitespace(c)) { + return new RwsState(c); + } else if(c == ')') { + return new CloseState(c); + } else if(isEof(c)) { + return finish(); + } else { + return new SearchTermState().init(c); + } + } + + @Override + public State init(char c) { + return nextChar(c); + } + } + + private class SearchTermState extends State { + public SearchTermState() { + super(Token.TERM); + } + @Override + public State nextChar(char c) { + if(c == 'n' || c == 'N') { + return new NotState(c); + } else if (c == '\'') { + return new SearchPhraseState(c); + } else if (isAllowedChar(c)) { + return new SearchWordState(c); + } else if (c == ')') { + finish(); + return new CloseState(c); + } else if (isWhitespace(c)) { + finish(); + return new RwsState(c); + } else if (isEof(c)) { + return finish(); + } + throw new IllegalStateException(this.getClass().getName() + "->" + c); + } + @Override + public State init(char c) { + return nextChar(c); + } + } + + private class SearchWordState extends State { + public SearchWordState(char c) { + super(Token.WORD, c); + } + public SearchWordState(State toConsume) { + super(Token.WORD, toConsume); + } + + @Override + public State nextChar(char c) { + // if(c == 'n' || c == 'N') { + // return new NotState(c); + // } + if (isAllowedChar(c)) { + return allowed(c); + } else if (c == ')') { + finish(); + return new CloseState(c); + } else if (isWhitespace(c)) { + finish(); + return new RwsState(c); + } else if (isEof(c)) { + return finish(); + } + throw new IllegalStateException(this.getClass().getName() + "->" + c); + } + } + + private class SearchPhraseState extends State { + public SearchPhraseState(char c) { + super(Token.PHRASE, c); + if(c != '\'') { + forbidden(c); + } + } + + @Override + public State nextChar(char c) { + if(isFinished() && !isEof(c)) { + return new SearchExpressionState().init(c); + } else if (isAllowedPhrase(c)) { + return allowed(c); + } else if (c == '\'') { + finish(); + return allowed(c); + } else if (isWhitespace(c)) { + if(isFinished()) { + return new RwsState(c); + } + return allowed(c); + } else if (isEof(c)) { + return finish(); + } + throw new IllegalStateException(this.getClass().getName() + "->" + c); + } + } + + private class OpenState extends State { + public OpenState(char c) { + super(Token.OPEN, c); + finish(); + } + @Override + public State nextChar(char c) { + finish(); + if (isWhitespace(c)) { + throw new IllegalStateException(this.getClass().getName() + "->" + c); + } + return new SearchExpressionState().init(c); + } + } + + private class CloseState extends State { + public CloseState(char c) { + super(Token.CLOSE, c); + finish(); + } + + @Override + public State nextChar(char c) { + if (isEof(c)) { + return finish(); + } else { + return new SearchExpressionState().init(c); + } + } + } + + private class NotState extends State { + public NotState(char c) { + super(Token.NOT, c); + } + @Override + public State nextChar(char c) { + if (getLiteral().length() == 1 && (c == 'o' || c == 'O')) { + return allowed(c); + } else if (getLiteral().length() == 2 && (c == 't' || c == 'T')) { + return allowed(c); + } else if(getLiteral().length() == 3 && isWhitespace(c)) { + finish(); + return new RwsState(c); + } else { + return new SearchWordState(this); + } + } + } + + private class AndState extends State { + public AndState(char c) { + super(Token.AND, c); + if(c != 'a' && c != 'A') { + forbidden(c); + } + } + @Override + public State nextChar(char c) { + if (getLiteral().length() == 1 && (c == 'n' || c == 'N')) { + return allowed(c); + } else if (getLiteral().length() == 2 && (c == 'd' || c == 'D')) { + return allowed(c); + } else if(getLiteral().length() == 3 && isWhitespace(c)) { + finish(); + return new RwsState(c); + } else { + return new SearchWordState(this); + } + } + } + + private class OrState extends State { + public OrState(char c) { + super(Token.OR, c); + if(c != 'o' && c != 'O') { + forbidden(c); + } + } + @Override + public State nextChar(char c) { + if (getLiteral().length() == 1 && (c == 'r' || c == 'R')) { + return allowed(c); + } else if(getLiteral().length() == 2 && isWhitespace(c)) { + finish(); + return new RwsState(c); + } else { + return new SearchWordState(this); + } + } + } + + + private class RwsState extends State { + public RwsState(char c) { + super(Token.RWS, c); + } + @Override + public State nextChar(char c) { + if (isWhitespace(c)) { + return allowed(c); + } else if (c == 'O' || c == 'o') { + return new OrState(c); + } else if (c == 'A' || c == 'a') { + return new AndState(c); + } else { + return new SearchExpressionState().init(c); + } + } + } + + // TODO (mibo): add (new) parse exception + public List<SearchQueryToken> tokenize(String searchQuery) { + char[] chars = searchQuery.toCharArray(); + + State state = new SearchExpressionState(); + List<SearchQueryToken> states = new ArrayList<SearchQueryToken>(); + for (char aChar : chars) { + State next = state.next(aChar); + if (state.isFinished() && next != state) { + states.add(state); + } + state = next; + } + + if(state.next(State.EOF).isFinished()) { + states.add(state); + } else { + throw new IllegalStateException("State: " + state + " not finished and list is: " + states.toString()); + } + + return states; + } + +} http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/queryoption/SearchOptionImpl.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/queryoption/SearchOptionImpl.java b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/queryoption/SearchOptionImpl.java index ec42147..51323a8 100644 --- a/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/queryoption/SearchOptionImpl.java +++ b/lib/server-core/src/main/java/org/apache/olingo/server/core/uri/queryoption/SearchOptionImpl.java @@ -24,13 +24,18 @@ import org.apache.olingo.server.api.uri.queryoption.search.SearchExpression; public class SearchOptionImpl extends SystemQueryOptionImpl implements SearchOption { + private SearchExpression searchExpression; + public SearchOptionImpl() { setKind(SystemQueryOptionKind.SEARCH); } @Override public SearchExpression getSearchExpression() { - return null; + return searchExpression; } + public void setSearchExpression(SearchExpression searchExpression) { + this.searchExpression = searchExpression; + } } http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java new file mode 100644 index 0000000..81147ba --- /dev/null +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchParserTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.olingo.server.core.uri.parser.search; + +import org.junit.Test; + +public class SearchParserTest { + + @Test + public void basicParsing() { + SearchParser parser = new SearchParser(); + parser.parse("ESAllPrim", "abc"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java ---------------------------------------------------------------------- diff --git a/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java new file mode 100644 index 0000000..15d35df --- /dev/null +++ b/lib/server-core/src/test/java/org/apache/olingo/server/core/uri/parser/search/SearchTokenizerTest.java @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.olingo.server.core.uri.parser.search; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import static org.apache.olingo.server.core.uri.parser.search.SearchQueryToken.Token.*; + +public class SearchTokenizerTest { + + private boolean logEnabled = false; + + @Test + public void testParse() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + // + result = tokenizer.tokenize("abc"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + + result = tokenizer.tokenize("NOT abc"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(NOT, result.get(0).getToken()); + Assert.assertEquals(WORD, result.get(1).getToken()); + + result = tokenizer.tokenize("(abc)"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(OPEN, result.get(0).getToken()); + Assert.assertEquals(WORD, result.get(1).getToken()); + Assert.assertEquals(CLOSE, result.get(2).getToken()); + + result = tokenizer.tokenize("((abc))"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(OPEN, result.get(0).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + Assert.assertEquals(CLOSE, result.get(4).getToken()); + } + + @Test + public void parseWords() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + // + result = tokenizer.tokenize("abc"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + + // + result = tokenizer.tokenize("9988abs"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + } + + @Test + public void parsePhrase() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + // + result = tokenizer.tokenize("'abc'"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(PHRASE, result.get(0).getToken()); + + // + result = tokenizer.tokenize("'9988 abs'"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(PHRASE, result.get(0).getToken()); + Assert.assertEquals("'9988 abs'", result.get(0).getLiteral()); + + // + result = tokenizer.tokenize("'99_88.'"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(PHRASE, result.get(0).getToken()); + Assert.assertEquals("'99_88.'", result.get(0).getLiteral()); + } + + @Test + public void testParseNot() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + result = tokenizer.tokenize("NOT abc"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(NOT, result.get(0).getToken()); + Assert.assertEquals(WORD, result.get(1).getToken()); + } + + @Test + public void testParseOr() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + result = tokenizer.tokenize("abc OR xyz"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(OR, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + + result = tokenizer.tokenize("abc OR xyz OR 123"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(OR, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + Assert.assertEquals(OR, result.get(3).getToken()); + Assert.assertEquals(WORD, result.get(4).getToken()); + } + + @Test + public void testParseAnd() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + result = tokenizer.tokenize("abc AND xyz"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + + result = tokenizer.tokenize("abc AND xyz AND 123"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + Assert.assertEquals(AND, result.get(3).getToken()); + Assert.assertEquals(WORD, result.get(4).getToken()); + + result = tokenizer.tokenize("abc AND 'x-y_z' AND 123"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(PHRASE, result.get(2).getToken()); + Assert.assertEquals("'x-y_z'", result.get(2).getLiteral()); + Assert.assertEquals(AND, result.get(3).getToken()); + Assert.assertEquals(WORD, result.get(4).getToken()); + } + + @Test + public void testParseAndOr() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + result = tokenizer.tokenize("abc AND xyz OR 123"); + Assert.assertNotNull(result); + log(result.toString()); + Assert.assertEquals(WORD, result.get(0).getToken()); + Assert.assertEquals(AND, result.get(1).getToken()); + Assert.assertEquals(WORD, result.get(2).getToken()); + Assert.assertEquals(OR, result.get(3).getToken()); + Assert.assertEquals(WORD, result.get(4).getToken()); + + SearchValidator.init("abc AND ANDsomething") + .addExpected(WORD, AND, WORD).validate(); + } + + + @Test + public void parseCombinations() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + + result = tokenizer.tokenize("abc AND NOT xyz OR 123"); + Assert.assertNotNull(result); + log(result.toString()); + Iterator<SearchQueryToken> it = result.iterator(); + Assert.assertEquals(WORD, it.next().getToken()); + Assert.assertEquals(AND, it.next().getToken()); + Assert.assertEquals(NOT, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + Assert.assertEquals(OR, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + + SearchValidator.init("foo AND bar OR foo AND baz OR that AND bar OR that AND baz") + .addExpected(WORD, "foo").addExpected(AND) + .addExpected(WORD, "bar").addExpected(OR) + .addExpected(WORD, "foo").addExpected(AND) + .addExpected(WORD, "baz").addExpected(OR) + .addExpected(WORD, "that").addExpected(AND) + .addExpected(WORD, "bar").addExpected(OR) + .addExpected(WORD, "that").addExpected(AND) + .addExpected(WORD, "baz") + .validate(); + + + SearchValidator.init("(foo OR that) AND (bar OR baz)").enableLogging() + .addExpected(OPEN) + .addExpected(WORD, "foo").addExpected(OR).addExpected(WORD, "that") + .addExpected(CLOSE).addExpected(AND).addExpected(OPEN) + .addExpected(WORD, "bar").addExpected(OR).addExpected(WORD, "baz") + .addExpected(CLOSE) + .validate(); + } + + + @Test + public void parseSpecial() throws Exception { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result; + Iterator<SearchQueryToken> it; + + result = tokenizer.tokenize("NOT abc AND nothing"); + log(result.toString()); + it = result.iterator(); + Assert.assertEquals(NOT, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + Assert.assertEquals(AND, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + + result = tokenizer.tokenize("abc AND andsomething"); + log(result.toString()); + it = result.iterator(); + Assert.assertEquals(WORD, it.next().getToken()); + Assert.assertEquals(AND, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + + result = tokenizer.tokenize("abc OR orsomething"); + log(result.toString()); + it = result.iterator(); + Assert.assertEquals(WORD, it.next().getToken()); + Assert.assertEquals(OR, it.next().getToken()); + Assert.assertEquals(WORD, it.next().getToken()); + + SearchValidator.init("abc AND ANDsomething") + .addExpected(WORD, AND, WORD).validate(); + } + + @Test + public void moreMixedTests() { + validate("abc"); + validate("NOT abc"); + + validate("abc AND def"); + validate("abc OR def"); + validate("abc def"); + + validate("abc AND def AND ghi", WORD, AND, WORD, AND, WORD); + validate("abc AND def OR ghi"); + validate("abc AND def ghi"); + + validate("abc OR def AND ghi"); + validate("abc OR def OR ghi"); + validate("abc OR def ghi"); + + validate("abc def AND ghi"); + validate("abc def OR ghi"); + validate("abc def ghi"); + + // mixed not + validate(" abc def AND ghi"); + validate("NOT abc NOT def OR NOT ghi"); + validate(" abc def NOT ghi"); + + // parenthesis + validate("(abc)"); + validate("(abc AND def)"); + validate("(abc AND def) OR ghi "); + validate("(abc AND def) ghi "); + validate("abc AND (def OR ghi)"); + validate("abc AND (def ghi)"); + } + + public boolean validate(String query) { + return new SearchValidator(query).validate(); + } + + public boolean validate(String query, SearchQueryToken.Token ... tokens) { + SearchValidator sv = new SearchValidator(query); + for (SearchQueryToken.Token token : tokens) { + sv.addExpected(token); + } + return sv.validate(); + } + + private static class SearchValidator { + private List<Tuple> validations = new ArrayList<Tuple>(); + private boolean log; + private final String searchQuery; + private class Tuple { + final SearchQueryToken.Token token; + final String literal; + public Tuple(SearchQueryToken.Token token, String literal) { + this.token = token; + this.literal = literal; + } + public Tuple(SearchQueryToken.Token token) { + this(token, null); + } + } + + private SearchValidator(String searchQuery) { + this.searchQuery = searchQuery; + } + + private static SearchValidator init(String searchQuery) { + return new SearchValidator(searchQuery); + } + private SearchValidator enableLogging() { + log = true; + return this; + } + private SearchValidator addExpected(SearchQueryToken.Token token, String literal) { + validations.add(new Tuple(token, literal)); + return this; + } + private SearchValidator addExpected(SearchQueryToken.Token ... token) { + for (SearchQueryToken.Token t : token) { + validations.add(new Tuple(t)); + } + return this; + } + private boolean validate() { + SearchTokenizer tokenizer = new SearchTokenizer(); + List<SearchQueryToken> result = tokenizer.tokenize(searchQuery); + Assert.assertNotNull(result); + if(log) { + System.out.println(result); + } + if(validations.size() != 0) { + Assert.assertEquals(validations.size(), result.size()); + + Iterator<Tuple> validationIt = validations.iterator(); + for (SearchQueryToken iToken : result) { + Tuple validation = validationIt.next(); + Assert.assertEquals(validation.token, iToken.getToken()); + if(validation.literal != null) { + Assert.assertEquals(validation.literal, iToken.getLiteral()); + } + } + } + + return true; + } + } + + + + private void log(Object ... toString) { + if(logEnabled) { + System.out.println("------------"); + if(toString == null || toString.length <= 1) { + System.out.println(toString == null? "NULL": (toString.length == 0? "EMPTY ARRAY": toString[0])); + } else { + int count = 1; + for (Object o : toString) { + System.out.println(count++ + ": " + o); + } + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/olingo-odata4/blob/452ebcbd/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java ---------------------------------------------------------------------- diff --git a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java index 6f1c4ad..259b61f 100644 --- a/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java +++ b/lib/server-test/src/test/java/org/apache/olingo/server/core/uri/antlr/TestFullResourcePath.java @@ -1165,8 +1165,9 @@ public class TestFullResourcePath { .isExValidation(UriValidationException.MessageKeys.SYSTEM_QUERY_OPTION_NOT_ALLOWED); // $search is currently not implemented. Please change this exception if the implementation is done. - testUri.runEx("FICRTCollETMixPrimCollCompTwoParam(ParameterInt16=1,ParameterString='1')", "$search=test") - .isExSemantic(MessageKeys.NOT_IMPLEMENTED); + // FIXME (151106:mibo): check after finish of OLINGO-568 +// testUri.runEx("FICRTCollETMixPrimCollCompTwoParam(ParameterInt16=1,ParameterString='1')", "$search=test") +// .isExSemantic(MessageKeys.NOT_IMPLEMENTED); testUri.run("ESAllPrim/olingo.odata.test1.BFNESAllPrimRTCTAllPrim()") .isKind(UriInfoKind.resource) @@ -5363,12 +5364,12 @@ public class TestFullResourcePath { testUri.run("ESTwoKeyNav", "$search= abc def NOT ghi"); // parenthesis - testUri.run("ESTwoKeyNav", "$search= (abc)"); - testUri.run("ESTwoKeyNav", "$search= (abc AND def)"); - testUri.run("ESTwoKeyNav", "$search= (abc AND def) OR ghi "); - testUri.run("ESTwoKeyNav", "$search= (abc AND def) ghi "); - testUri.run("ESTwoKeyNav", "$search= abc AND (def OR ghi)"); - testUri.run("ESTwoKeyNav", "$search= abc AND (def ghi)"); + testUri.run("ESTwoKeyNav", "$search=(abc)"); + testUri.run("ESTwoKeyNav", "$search=(abc AND def)"); + testUri.run("ESTwoKeyNav", "$search=(abc AND def) OR ghi "); + testUri.run("ESTwoKeyNav", "$search=(abc AND def) ghi "); + testUri.run("ESTwoKeyNav", "$search=abc AND (def OR ghi)"); + testUri.run("ESTwoKeyNav", "$search=abc AND (def ghi)"); } @Test
