Add numeric string tests BugzId: 45572
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/ca5ff7bc Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/ca5ff7bc Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/ca5ff7bc Branch: refs/heads/master Commit: ca5ff7bcb33e6242b3fa4bdafc40955d984ca439 Parents: a8def14 Author: Tony Sun <tony....@cloudant.com> Authored: Tue Aug 25 10:51:22 2015 -0700 Committer: Tony Sun <tony....@cloudant.com> Committed: Tue Aug 25 13:23:52 2015 -0700 ---------------------------------------------------------------------- src/mango_selector_text.erl | 13 +++++++++-- src/mango_util.erl | 43 +++++++++++++++++++++++++---------- test/06-basic-text-test.py | 30 ++++++++++++++++++++++++ test/literal_gen.py | 49 ++++++++++++++++++++++++++++++++++++++++ test/mango.py | 8 +++++++ test/num_string_docs.py | 49 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/src/mango_selector_text.erl ---------------------------------------------------------------------- diff --git a/src/mango_selector_text.erl b/src/mango_selector_text.erl index 47e25ce..74a4012 100644 --- a/src/mango_selector_text.erl +++ b/src/mango_selector_text.erl @@ -45,8 +45,8 @@ convert(Path, {[{<<"$default">>, Arg}]}) -> % The $text operator specifies a Lucene syntax query % so we just pull it in directly. convert(Path, {[{<<"$text">>, Query}]}) when is_binary(Query) -> - Term = mango_util:append_quotes(value_str(Query)), - {op_field, {make_field(Path, Query), Term}}; + Value = maybe_append_quotes(value_str(Query)), + {op_field, {make_field(Path, Query), Value}}; % The MongoDB docs for $all are super confusing and read more % like they screwed up the implementation of this operator @@ -316,6 +316,15 @@ append_sort_type(RawSortField, Selector) -> end. +maybe_append_quotes(TextValue) -> + case mango_util:is_number_string(TextValue) of + true -> + <<"\"", TextValue/binary, "\"">>; + false -> + TextValue + end. + + get_sort_type(Field, Selector) -> Types = get_sort_types(Field, Selector, []), case lists:usort(Types) of http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/src/mango_util.erl ---------------------------------------------------------------------- diff --git a/src/mango_util.erl b/src/mango_util.erl index b12c6e9..fd572c7 100644 --- a/src/mango_util.erl +++ b/src/mango_util.erl @@ -34,7 +34,7 @@ lucene_escape_field/1, lucene_escape_query_value/1, lucene_escape_user/1, - append_quotes/1, + is_number_string/1, has_suffix/2, @@ -46,7 +46,33 @@ -include("mango.hrl"). -%% https://gist.github.com/cad2d2456c518d878d08 +%% Regex for Floating Point, Hex Floating Point, Nan, and Infinity +%% Digits = "(\\p{N}+)", +%% HexDigits = "([0-9a-fA-F]+)", +%% Exp = "[eE][+-]?" ++ Digits, +%% FpRegexStr = "[\\x00-\\x20]*" ++ "[+-]?(" ++ "NaN|" +%% ++ "Infinity|" ++ "(((" +%% ++ Digits +%% ++ "(\\.)?(" +%% ++ Digits +%% ++ "?)(" +%% ++ Exp +%% ++ ")?)|" +%% ++ "(\\.(" +%% ++ Digits +%% ++ ")(" +%% ++ Exp +%% ++ ")?)|" +%% ++ "((" +%% ++ "(0[xX]" +%% ++ HexDigits +%% ++ "(\\.)?)|" +%% ++ "(0[xX]" +%% ++ HexDigits +%% ++ "?(\\.)" +%% ++ HexDigits +%% ++ ")" +%% ++ ")[pP][+-]?" ++ Digits ++ "))" ++ "[fFdD]?))" ++ "[\\x00-\\x20]*" -define(NUMSTRING, {re_pattern,25,0,<<69,82,67,80,71,3,0,0,0,0,0,0,1,0,0,0,25,0, 0,0,0,0,0,0,48,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93,3,19,77,255,255, 255,255,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,69,77,0,0,0,0,0, @@ -342,19 +368,12 @@ join(Sep, [Item | Rest]) -> [Item, Sep | join(Sep, Rest)]. -is_number_string(Subject) -> +is_number_string(Value) when is_binary(Value) -> + is_number_string(binary_to_list(Value)); +is_number_string(Value) when is_list(Value)-> case re:run(Subject, ?NUMSTRING) of nomatch -> false; _ -> true end. - - -append_quotes(TextValue) -> - case is_number_string(binary_to_list(TextValue)) of - true -> - <<"\"", TextValue/binary, "\"">>; - false -> - TextValue - end. http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/test/06-basic-text-test.py ---------------------------------------------------------------------- diff --git a/test/06-basic-text-test.py b/test/06-basic-text-test.py index 08739db..baf912b 100644 --- a/test/06-basic-text-test.py +++ b/test/06-basic-text-test.py @@ -14,6 +14,7 @@ import json import mango import unittest import user_docs +import num_string_docs class TextIndexCheckTests(mango.DbPerClass): @@ -511,3 +512,32 @@ class ElemMatchTests(mango.FriendDocsTextTests): assert len(docs) == 3 for d in docs: assert d["user_id"] in (10, 11,12) + + +# Test numeric strings for $text +class NumStringTests(mango.NumStringDocsTextTests): + def test_floating_point_val(self): + float_point_string = num_string_docs.DOCS[2]["number_string"] + q = {"$text": float_point_string} + docs = self.db.find(q) + assert len(docs) == 1 + assert docs[0]["number_string"] == float_point_string + + def test_hex_floating_point_val(self): + hex_float_point_string = num_string_docs.DOCS[3]["number_string"] + q = {"$text": hex_float_point_string} + docs = self.db.find(q) + assert len(docs) == 1 + assert docs[0]["number_string"] == hex_float_point_string + + def test_nan_val(self): + q = {"$text": "NaN"} + docs = self.db.find(q) + assert len(docs) == 1 + assert docs[0]["number_string"] == "NaN" + + def test_infinity_val(self): + q = {"$text": "Infinity"} + docs = self.db.find(q) + assert len(docs) == 1 + assert docs[0]["number_string"] == "Infinity" http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/test/literal_gen.py ---------------------------------------------------------------------- diff --git a/test/literal_gen.py b/test/literal_gen.py new file mode 100644 index 0000000..7cf60dd --- /dev/null +++ b/test/literal_gen.py @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import random +from string import digits, hexdigits + +def joiner(*items): + return ''.join(item() for item in items) + +def roll(item, n1, n2=None): + n2 = n2 or n1 + return lambda: ''.join(item() for _ in xrange(random.randint(n1, n2))) + +def rand(collection): + return lambda: random.choice(collection) + +def floating_point_literal(): + return joiner(roll(rand(' '), 0, 2), + roll(rand('+-'), 0, 1), + roll(rand(digits), 2, 20), + rand('.'), + roll(rand(digits), 2, 20), + roll(rand('eE'), 1, 1), + roll(rand('+-'), 0, 1), + roll(rand(digits), 2, 20), + roll(rand('fF'), 0, 1), + roll(rand(' '), 0, 2)) + +def hex_floating_point_literal(): + return joiner(roll(rand(' '), 0, 2), + rand('0'), + roll(rand('xX'), 1, 1), + roll(rand(hexdigits), 1, 6), + rand('.'), + roll(rand(hexdigits), 1, 7), + roll(rand('pP'), 1, 1), + roll(rand('+-'), 0, 1), + roll(rand(digits), 1, 4), + roll(rand('fFdD'), 0, 1), + roll(rand(' '), 0, 2)) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/test/mango.py ---------------------------------------------------------------------- diff --git a/test/mango.py b/test/mango.py index bde9323..6764ed5 100644 --- a/test/mango.py +++ b/test/mango.py @@ -20,6 +20,7 @@ import requests import friend_docs import user_docs import limit_docs +import num_string_docs def random_db_name(): @@ -235,3 +236,10 @@ class LimitDocsTextTests(DbPerClass): def setUpClass(klass): super(LimitDocsTextTests, klass).setUpClass() limit_docs.setup(klass.db, index_type="text") + +class NumStringDocsTextTests(DbPerClass): + + @classmethod + def setUpClass(klass): + super(NumStringDocsTextTests, klass).setUpClass() + num_string_docs.setup(klass.db, index_type="text") http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/ca5ff7bc/test/num_string_docs.py ---------------------------------------------------------------------- diff --git a/test/num_string_docs.py b/test/num_string_docs.py new file mode 100644 index 0000000..f900120 --- /dev/null +++ b/test/num_string_docs.py @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +import copy +import literal_gen + + +def setup(db, index_type="view"): + db.recreate() + DOCS[2]["number_string"] = literal_gen.floating_point_literal() + DOCS[3]["number_string"] = literal_gen.hex_floating_point_literal() + db.save_docs(copy.deepcopy(DOCS)) + if index_type == "view": + add_view_indexes(db) + elif index_type == "text": + add_text_indexes(db) + + +def add_text_indexes(db): + db.create_text_index() + + +DOCS = [ + { + "_id": "55118b87283f8f2901c59663", + "number_string": "NaN" + }, + { + "_id": "55118b873c98123d69bff407", + "number_string": "Infinity" + }, + { + "_id": "55118b87b4e99951e6fbe5c4", + "number_string": "filler" + }, + { + "_id": "55118b87bc21952536ef00da", + "number_string": "filler" + } +] \ No newline at end of file