Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983524
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2,
normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally
used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the
container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should
be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName,
FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8),
ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len,
att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored
length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data,
NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully
~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <-
Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur,
KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding,
disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk
len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode,
utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original
length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 =
couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
--- End diff --
I also think that external is not the best choice. Since field is already
carries external word in the key, better specify which exact external we assume
here. Today it's only swift, tomorrow it could be also S3/NFS/whatever which
will have to use different integration logic.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
