[
https://issues.apache.org/jira/browse/COUCHDB-769?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14956756#comment-14956756
]
ASF GitHub Bot commented on COUCHDB-769:
----------------------------------------
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983028
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2,
normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally
used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the
container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should
be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName,
FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8),
ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len,
att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored
length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data,
NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully
~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <-
Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur,
KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding,
disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk
len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode,
utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original
length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 =
couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
+ NewAtt1;
+ {_,{Code,_}} ->
+ couch_log:debug("Swift: response code is ~p~n",[Code]),
+ Att
+ end;
+ _ ->
+ Att
+ end.
+
+swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) ->
+ couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container,
ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}],
+ Method = put,
+ couch_log:debug("Swift: going to upload : ~p ~p ~p ~p
~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url: ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Headers,
unicode:characters_to_list(ContextType), Data}, [], []),
+ couch_log:debug("~p~n",[R]),
+ case R of
+ {ok,_} ->
+ {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R,
+ {ok,{ReturnCode,NewUrl}};
+ Error ->
+ {error,{element(2,Error),NewUrl}}
+ end;
+ {not_authenticated,_} ->
+ {error,"Not authenticated",""};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_delete_object(Container, ObjName) ->
+ couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container,
ObjName]).
+
+swift_get_object(Container, ObjName) ->
+ couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ Method = get,
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R,
+ couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]),
+ Body;
+ {not_authenticated,_} ->
+ {error, "Not authenticated"};
+ {not_supported, Error} ->
+ {error,{element(2,Error),""}}
+ end.
+
+swift_head_object(Container, ObjName) ->
+ couch_log:debug("Swift: head object ~p/~p ~n",[Container, ObjName]),
+ case authenticate() of
+ {ok,{Url,Storage_Token}} ->
+ Header = [{"X-Auth-Token",Storage_Token}],
+ Method = head,
+ ObjNameEncoded = couch_util:url_encode(ObjName),
+ NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container)
++ "/" ++ unicode:characters_to_list(ObjNameEncoded),
+ couch_log:debug("Swift: url ~p~n",[NewUrl]),
+ R = httpc:request(Method, {NewUrl, Header}, [], []),
+ {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R,
+ couch_log:debug("Swift: ~p~p~n",[ReturnCode, Head]),
+ ObjectSize = lists:filter(fun ({"content-length",_}) -> true ;
(_) -> false end, Head),
+ EtagHeader = lists:filter(fun ({"etag",_}) -> true ; (_) ->
false end, Head),
+ Etag = element(2,lists:nth(1,EtagHeader)),
+ {ObjectSizeNumeric,_} =
string:to_integer(element(2,lists:nth(1,ObjectSize))),
+ couch_log:debug("Swift: Object size is: ~p and Etag:
~p~n",[ObjectSizeNumeric, Etag]),
+ couch_log:debug("Etag in base16 ~p~n",[Etag]),
+ EtagDecode = hex_to_bin(Etag),
+ EtagMD5 = base64:encode(EtagDecode),
+ couch_log:debug("Etag in base64 ~p~n",[EtagMD5]),
+ {ObjectSizeNumeric,EtagMD5};
+ {auth_not_supported, Error} ->
+ {error,{element(2,Error),""}};
+ {not_authenticated, _} ->
+ {-1,""}
--- End diff --
What -1 stand for?
> Store large attachments external to the .couch file
> ---------------------------------------------------
>
> Key: COUCHDB-769
> URL: https://issues.apache.org/jira/browse/COUCHDB-769
> Project: CouchDB
> Issue Type: New Feature
> Components: Database Core
> Reporter: Robert Newson
> Assignee: Adam Kocoloski
> Attachments: external_attachments_alpha.patch
>
>
> For attachment-heavy applications storing the attachments in separate files
> significantly eases compaction problems.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
