Repository: couchdb-mango
Updated Branches:
refs/heads/master 677cd2a59 -> 1de64eabb
Revert "Remove reference to _text indexes"
This reverts commit 955a42c3bbd0502f1623bec29edad59eddd7b2ea.
Conflicts:
src/mango_error.erl
COUCHDB-2787
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mango/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mango/commit/fc1e36f3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mango/tree/fc1e36f3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mango/diff/fc1e36f3
Branch: refs/heads/master
Commit: fc1e36f354475981a4fce044a160f14251922421
Parents: 677cd2a
Author: Tony Sun <tony....@cloudant.com>
Authored: Mon Aug 24 14:17:01 2015 -0700
Committer: Tony Sun <tony....@cloudant.com>
Committed: Mon Aug 24 14:17:01 2015 -0700
----------------------------------------------------------------------
src/mango_cursor.erl | 3 +-
src/mango_cursor_text.erl | 307 +++++++++++++++++++++++++++++++++++++++++
src/mango_error.erl | 39 ++++++
src/mango_idx.erl | 10 +-
4 files changed, 355 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_cursor.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor.erl b/src/mango_cursor.erl
index 0b58c12..545a863 100644
--- a/src/mango_cursor.erl
+++ b/src/mango_cursor.erl
@@ -124,7 +124,8 @@ group_indexes_by_type(Indexes) ->
% don't suddenly switch indexes for existing client
% queries.
CursorModules = [
- mango_cursor_view
+ mango_cursor_view,
+ mango_cursor_text
],
lists:flatmap(fun(CMod) ->
case dict:find(CMod, IdxDict) of
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_cursor_text.erl
----------------------------------------------------------------------
diff --git a/src/mango_cursor_text.erl b/src/mango_cursor_text.erl
new file mode 100644
index 0000000..c774c82
--- /dev/null
+++ b/src/mango_cursor_text.erl
@@ -0,0 +1,307 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(mango_cursor_text).
+
+-export([
+ create/4,
+ explain/1,
+ execute/3
+]).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("dreyfus/include/dreyfus.hrl").
+-include("mango_cursor.hrl").
+-include("mango.hrl").
+
+
+-record(cacc, {
+ selector,
+ dbname,
+ ddocid,
+ idx_name,
+ query_args,
+ bookmark,
+ limit,
+ skip,
+ user_fun,
+ user_acc
+}).
+
+
+create(Db, Indexes, Selector, Opts0) ->
+ Index = case Indexes of
+ [Index0] ->
+ Index0;
+ _ ->
+ ?MANGO_ERROR(multiple_text_indexes)
+ end,
+
+ Opts = unpack_bookmark(Db#db.name, Opts0),
+
+ % Limit the result set size to 50 for Clouseau's
+ % sake. We may want to revisit this.
+ Limit0 = couch_util:get_value(limit, Opts, 50),
+ Limit = if Limit0 < 50 -> Limit0; true -> 50 end,
+ Skip = couch_util:get_value(skip, Opts, 0),
+ Fields = couch_util:get_value(fields, Opts, all_fields),
+
+ {ok, #cursor{
+ db = Db,
+ index = Index,
+ ranges = null,
+ selector = Selector,
+ opts = Opts,
+ limit = Limit,
+ skip = Skip,
+ fields = Fields
+ }}.
+
+
+explain(Cursor) ->
+ #cursor{
+ selector = Selector,
+ opts = Opts
+ } = Cursor,
+ [
+ {'query', mango_selector_text:convert(Selector)},
+ {sort, sort_query(Opts, Selector)}
+ ].
+
+
+execute(Cursor, UserFun, UserAcc) ->
+ #cursor{
+ db = Db,
+ index = Idx,
+ limit = Limit,
+ skip = Skip,
+ selector = Selector,
+ opts = Opts
+ } = Cursor,
+ QueryArgs = #index_query_args{
+ q = mango_selector_text:convert(Selector),
+ sort = sort_query(Opts, Selector),
+ raw_bookmark = true
+ },
+ CAcc = #cacc{
+ selector = Selector,
+ dbname = Db#db.name,
+ ddocid = ddocid(Idx),
+ idx_name = mango_idx:name(Idx),
+ bookmark = get_bookmark(Opts),
+ limit = Limit,
+ skip = Skip,
+ query_args = QueryArgs,
+ user_fun = UserFun,
+ user_acc = UserAcc
+ },
+ try
+ execute(CAcc)
+ catch
+ throw:{stop, FinalCAcc} ->
+ #cacc{
+ bookmark = FinalBM,
+ user_fun = UserFun,
+ user_acc = LastUserAcc
+ } = FinalCAcc,
+ JsonBM = dreyfus_bookmark:pack(FinalBM),
+ Arg = {add_key, bookmark, JsonBM},
+ {_Go, FinalUserAcc} = UserFun(Arg, LastUserAcc),
+ {ok, FinalUserAcc}
+ end.
+
+
+execute(CAcc) ->
+ case search_docs(CAcc) of
+ {ok, Bookmark, []} ->
+ % If we don't have any results from the
+ % query it means the request has paged through
+ % all possible results and the request is over.
+ NewCAcc = CAcc#cacc{bookmark = Bookmark},
+ throw({stop, NewCAcc});
+ {ok, Bookmark, Hits} ->
+ NewCAcc = CAcc#cacc{bookmark = Bookmark},
+ HitDocs = get_json_docs(CAcc#cacc.dbname, Hits),
+ {ok, FinalCAcc} = handle_hits(NewCAcc, HitDocs),
+ execute(FinalCAcc)
+ end.
+
+
+search_docs(CAcc) ->
+ #cacc{
+ dbname = DbName,
+ ddocid = DDocId,
+ idx_name = IdxName
+ } = CAcc,
+ QueryArgs = update_query_args(CAcc),
+ case dreyfus_fabric_search:go(DbName, DDocId, IdxName, QueryArgs) of
+ {ok, Bookmark, _, Hits, _, _} ->
+ {ok, Bookmark, Hits};
+ {error, Reason} ->
+ ?MANGO_ERROR({text_search_error, {error, Reason}})
+ end.
+
+
+handle_hits(CAcc, []) ->
+ {ok, CAcc};
+
+handle_hits(CAcc0, [{Sort, Doc} | Rest]) ->
+ CAcc1 = handle_hit(CAcc0, Sort, Doc),
+ handle_hits(CAcc1, Rest).
+
+
+handle_hit(CAcc0, Sort, Doc) ->
+ #cacc{
+ limit = Limit,
+ skip = Skip
+ } = CAcc0,
+ CAcc1 = update_bookmark(CAcc0, Sort),
+ case mango_selector:match(CAcc1#cacc.selector, Doc) of
+ true when Skip > 0 ->
+ CAcc1#cacc{skip = Skip - 1};
+ true when Limit == 0 ->
+ % We hit this case if the user spcified with a
+ % zero limit. Notice that in this case we need
+ % to return the bookmark from before this match
+ throw({stop, CAcc0});
+ true when Limit == 1 ->
+ NewCAcc = apply_user_fun(CAcc1, Doc),
+ throw({stop, NewCAcc});
+ true when Limit > 1 ->
+ NewCAcc = apply_user_fun(CAcc1, Doc),
+ NewCAcc#cacc{limit = Limit - 1};
+ false ->
+ CAcc1
+ end.
+
+
+apply_user_fun(CAcc, Doc) ->
+ #cacc{
+ user_fun = UserFun,
+ user_acc = UserAcc
+ } = CAcc,
+ case UserFun({row, Doc}, UserAcc) of
+ {ok, NewUserAcc} ->
+ CAcc#cacc{user_acc = NewUserAcc};
+ {stop, NewUserAcc} ->
+ throw({stop, CAcc#cacc{user_acc = NewUserAcc}})
+ end.
+
+
+%% Convert Query to Dreyfus sort specifications
+%% Covert <<"Field">>, <<"desc">> to <<"-Field">>
+%% and append to the dreyfus query
+sort_query(Opts, Selector) ->
+ {sort, {Sort}} = lists:keyfind(sort, 1, Opts),
+ SortList = lists:map(fun(SortField) ->
+ {Dir, RawSortField} = case SortField of
+ {Field, <<"asc">>} -> {asc, Field};
+ {Field, <<"desc">>} -> {desc, Field};
+ Field when is_binary(Field) -> {asc, Field}
+ end,
+ SField = mango_selector_text:append_sort_type(RawSortField, Selector),
+ case Dir of
+ asc ->
+ SField;
+ desc ->
+ <<"-", SField/binary>>
+ end
+ end, Sort),
+ case SortList of
+ [] -> relevance;
+ _ -> SortList
+ end.
+
+
+get_bookmark(Opts) ->
+ case lists:keyfind(bookmark, 1, Opts) of
+ {_, BM} when is_list(BM), BM /= [] ->
+ BM;
+ _ ->
+ nil
+ end.
+
+
+update_bookmark(CAcc, Sortable) ->
+ BM = CAcc#cacc.bookmark,
+ QueryArgs = CAcc#cacc.query_args,
+ Sort = QueryArgs#index_query_args.sort,
+ NewBM = dreyfus_bookmark:update(Sort, BM, [Sortable]),
+ CAcc#cacc{bookmark = NewBM}.
+
+
+pack_bookmark(Bookmark) ->
+ case dreyfus_bookmark:pack(Bookmark) of
+ null -> nil;
+ Enc -> Enc
+ end.
+
+
+unpack_bookmark(DbName, Opts) ->
+ NewBM = case lists:keyfind(bookmark, 1, Opts) of
+ {_, nil} ->
+ [];
+ {_, Bin} ->
+ try
+ dreyfus_bookmark:unpack(DbName, Bin)
+ catch _:_ ->
+ ?MANGO_ERROR({invalid_bookmark, Bin})
+ end
+ end,
+ lists:keystore(bookmark, 1, Opts, {bookmark, NewBM}).
+
+
+ddocid(Idx) ->
+ case mango_idx:ddoc(Idx) of
+ <<"_design/", Rest/binary>> ->
+ Rest;
+ Else ->
+ Else
+ end.
+
+
+update_query_args(CAcc) ->
+ #cacc{
+ bookmark = Bookmark,
+ query_args = QueryArgs
+ } = CAcc,
+ QueryArgs#index_query_args{
+ bookmark = pack_bookmark(Bookmark),
+ limit = get_limit(CAcc)
+ }.
+
+
+get_limit(CAcc) ->
+ Total = CAcc#cacc.limit + CAcc#cacc.skip,
+ if
+ Total < 25 -> 25;
+ Total > 100 -> 100;
+ true -> Total
+ end.
+
+
+get_json_docs(DbName, Hits) ->
+ Ids = lists:map(fun(#sortable{item = Item}) ->
+ couch_util:get_value(<<"_id">>, Item#hit.fields)
+ end, Hits),
+ {ok, IdDocs} = dreyfus_fabric:get_json_docs(DbName, Ids),
+ lists:map(fun(#sortable{item = Item} = Sort) ->
+ Id = couch_util:get_value(<<"_id">>, Item#hit.fields),
+ case lists:keyfind(Id, 1, IdDocs) of
+ {Id, {doc, Doc}} ->
+ {Sort, Doc};
+ false ->
+ {Sort, not_found}
+ end
+ end, Hits).
+
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_error.erl
----------------------------------------------------------------------
diff --git a/src/mango_error.erl b/src/mango_error.erl
index 69767cf..cf117ab 100644
--- a/src/mango_error.erl
+++ b/src/mango_error.erl
@@ -46,6 +46,32 @@ info(mango_cursor, {no_usable_index, selector_unsupported})
->
<<"There is no index available for this selector.">>
};
+info(mango_cursor_text, {invalid_bookmark, BadBookmark}) ->
+ {
+ 400,
+ <<"invalid_bookmark">>,
+ fmt("Invalid boomkark value: ~s", [?JSON_ENCODE(BadBookmark)])
+ };
+info(mango_cursor_text, multiple_text_indexes) ->
+ {
+ 400,
+ <<"multiple_text_indexes">>,
+ <<"You must specify an index with the `use_index` parameter.">>
+ };
+info(mango_cursor_text, {text_search_error, {error, {bad_request, Msg}}})
+ when is_binary(Msg) ->
+ {
+ 400,
+ <<"text_search_error">>,
+ Msg
+ };
+info(mango_cursor_text, {text_search_error, {error, Error}}) ->
+ {
+ 400,
+ <<"text_search_error">>,
+ fmt("Error performing text search: ~p", [Error])
+ };
+
info(mango_fields, {invalid_fields_json, BadFields}) ->
{
400,
@@ -122,6 +148,19 @@ info(mango_idx_view, {index_not_found, BadIdx}) ->
fmt("JSON index ~s not found in this design doc.", [BadIdx])
};
+info(mango_idx_text, {invalid_index_text, BadIdx}) ->
+ {
+ 400,
+ <<"invalid_index">>,
+ fmt("Text indexes must be an object, not: ~w", [BadIdx])
+ };
+info(mango_idx_text, {index_not_found, BadIdx}) ->
+ {
+ 404,
+ <<"index_not_found">>,
+ fmt("Text index ~s not found in this design doc.", [BadIdx])
+ };
+
info(mango_opts, {invalid_bulk_docs, Val}) ->
{
400,
http://git-wip-us.apache.org/repos/asf/couchdb-mango/blob/fc1e36f3/src/mango_idx.erl
----------------------------------------------------------------------
diff --git a/src/mango_idx.erl b/src/mango_idx.erl
index f6e688b..71a55a9 100644
--- a/src/mango_idx.erl
+++ b/src/mango_idx.erl
@@ -167,7 +167,7 @@ from_ddoc(Db, {Props}) ->
?MANGO_ERROR(invalid_query_ddoc_language)
end,
- IdxMods = [mango_idx_view],
+ IdxMods = [mango_idx_view, mango_idx_text],
Idxs = lists:flatmap(fun(Mod) -> Mod:from_ddoc({Props}) end, IdxMods),
lists:map(fun(Idx) ->
Idx#idx{
@@ -241,13 +241,17 @@ end_key(#idx{}=Idx, Ranges) ->
cursor_mod(#idx{type = <<"json">>}) ->
mango_cursor_view;
cursor_mod(#idx{def = all_docs, type= <<"special">>}) ->
- mango_cursor_view.
+ mango_cursor_view;
+cursor_mod(#idx{type = <<"text">>}) ->
+ mango_cursor_text.
idx_mod(#idx{type = <<"json">>}) ->
mango_idx_view;
idx_mod(#idx{type = <<"special">>}) ->
- mango_idx_special.
+ mango_idx_special;
+idx_mod(#idx{type = <<"text">>}) ->
+ mango_idx_text.
db_to_name(#db{name=Name}) ->
