[
https://issues.apache.org/jira/browse/COUCHDB-769?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14956766#comment-14956766
]
ASF GitHub Bot commented on COUCHDB-769:
----------------------------------------
Github user kxepal commented on a diff in the pull request:
https://github.com/apache/couchdb-fabric/pull/33#discussion_r41983311
--- Diff: src/fabric_attachments_handler.erl ---
@@ -0,0 +1,333 @@
+%% @author gilv
+%% @doc @todo Add description to fabric_swift_handler.
+
+
+-module(fabric_attachments_handler).
+
+-include_lib("fabric/include/fabric.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-export([inline_att_store/2, inline_att_handler/3, container_handler/2,
normal_att_store/5, externalize_att/1]).
+
+
+%% ====================================================================
+%% External API functions. I try to keep them as general as possible,
+%% without correlation to specific object store that might be internally
used.
+%% ====================================================================
+
+normal_att_store(FileName,Db,ContentLen,MimeType,Req) ->
+ ContainerName = container_name(Db),
+ couch_log:debug("Standard attachment handler",[]),
+ couch_log:debug("Going to store ~p of length is ~p, ~p in the
container: ~n",[FileName,ContentLen,ContainerName]),
+ %Bad implementation - no chunk reader. All kept in the memory. Should
be fixed.
+ Data = couch_httpd:recv(Req, ContentLen),
+ case swift_put_object(ContainerName, FileName, MimeType, [], Data) of
+ {ok,{201,NewUrl}} ->
+ couch_log:debug("Created. Swift response code is 201 ~n",[]),
+ {ObjectSize,EtagMD5} = swift_head_object(ContainerName,
FileName),
+ {unicode:characters_to_binary(NewUrl, unicode, utf8),
ObjectSize, EtagMD5};
+ {_,{Code,_}} ->
+ couch_log:debug("Swift response code is ~p~n",[]),
+ {Data, -1}
+ end.
+
+inline_att_store(Db, Docs) ->
+ DbName = container_name(Db),
+ couch_log:debug("Store inline base64 encoded attachment ~n",[]),
+ if
+ is_list(Docs) ->
+ couch_log:debug("Going to handle document list",[]),
+ DocsArray = Docs;
+ true ->
+ couch_log:debug("Going to handle single document",[]),
+ DocsArray = [Docs]
+ end,
+ [#doc{atts=Atts0} = Doc | Rest] = DocsArray,
+ if
+ length(Atts0) > 0 ->
+ couch_log:debug("Att length is larger than 0",[]),
+ Atts = [att_processor(DbName,Att) || Att <- Atts0],
+ if
+ is_list(Docs) ->
+ [Doc#doc{atts = Atts} | Rest];
+ true ->
+ Doc#doc{atts = Atts}
+ end;
+ true ->
+ couch_log:debug("No attachments to handle",[]),
+ Docs
+ end.
+
+inline_att_handler(get, Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Retrieve attachment",[]),
+ [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len,
att_external_size],Att),
+ couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored
length: ~p~n",[Name, AttLen, AttLenUser]),
+ NewData = swift_get_object(DbName, Name),
+ NewAtt = couch_att:store([{data,
NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att),
+ NewAtt;
+
+inline_att_handler(delete,Db, Att) ->
+ DbName = container_name(Db),
+ couch_log:debug("Delete attachment ~p~n",[]).
+
+container_handler(create,DbName) ->
+ couch_log:debug("Create container ~p~n",[DbName]),
+ case swift_create_container(DbName) of
+ {ok,{201,Container}} ->
+ couch_log:debug("Container ~p created succesfully
~n",[Container]),
+ {ok,Container};
+ {error,_} ->
+ couch_log:debug("Container ~p creation failed ~n",[DbName]),
+ {error,DbName}
+ end;
+container_handler(get,DbName) ->
+ couch_log:debug("Get container ~p~n",[DbName]);
+container_handler(delete,DbName)->
+ couch_log:debug("Delete container ~p~n",[DbName]),
+ swift_delete_container(DbName).
+
+externalize_att(Db) ->
+ Res = config:get("swift","attachments_offload","false"),
+ Res.
+
+%% ====================================================================
+%% Internal general functions
+%% ====================================================================
+
+container_name(Db) ->
+ Suffix = list_to_binary(mem3:shard_suffix(Db)),
+ DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]),
+ couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]),
+ DbNameSuffix.
+
+hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <-
Str >>.
+
+%% ====================================================================
+%% Internal OpenStack Swift implementation
+%% ====================================================================
+
+extractAuthInfo([{_,_}|_] = Obj,[])->
+ Storage_URL = proplists:get_value(<<"publicURL">>, Obj),
+ Storage_Token = proplists:get_value(<<"id">>, Obj),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{[{_,_}|_]}] = Obj,[])->
+ [{ObjNext}] = Obj,
+ Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext),
+ Storage_Token = proplists:get_value(<<"id">>, ObjNext),
+ Status = true,
+ [Storage_URL, Storage_Token, Status];
+extractAuthInfo([{_,_}|L]=Obj,KeyValPath)->
+ [{Key, Val}|KeyValPathNext] = KeyValPath,
+ ObjVal = proplists:get_value(Key,Obj),
+ case Val of
+ [] -> extractAuthInfo(ObjVal, KeyValPathNext);
+ ObjVal -> extractAuthInfo(Obj, KeyValPathNext);
+ _ -> ["", "", false]
+ end;
+extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)->
+ [ObjCur| _] = Obj,
+ [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur,
KeyValPath),
+ case Status of
+ false -> extractAuthInfo(L, KeyValPath);
+ _ -> [Storage_URL, Storage_Token, Status]
+ end;
+extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)->
+ {Doc} = Obj,
+ extractAuthInfo(Doc, KeyValPath).
+
+att_processor(DbName,Att) ->
+ couch_log:debug("Swift: attachment processor",[]),
+ [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding,
disk_len, att_len], Att),
+ [Name, Data] = couch_att:fetch([name, data], Att),
+ couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk
len: ~p~n",[Name,Type,Enc,DiskLen]),
+ case is_binary(Data) of
+ true ->
+ couch_log:debug("Swift: binary attachment exists",[]),
+ case swift_put_object(DbName, Name, Type, [], Data) of
+ {ok,{201,NewUrl}} ->
+ N1 = unicode:characters_to_binary(NewUrl, unicode,
utf8),
+ couch_log:debug("~p ~p~n",[N1, NewUrl]),
+ NewAtt = couch_att:store(data,N1,Att),
+ couch_log:debug("Swift. testing store in original
length ~p~n",[AttLen]),
+ {ObjectSize,EtagMD5} = swift_head_object(DbName, Name),
+ NewAtt1 =
couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt),
--- End diff --
Why you store "external" as list, no at atom, or at least as binary?
> Store large attachments external to the .couch file
> ---------------------------------------------------
>
> Key: COUCHDB-769
> URL: https://issues.apache.org/jira/browse/COUCHDB-769
> Project: CouchDB
> Issue Type: New Feature
> Components: Database Core
> Reporter: Robert Newson
> Assignee: Adam Kocoloski
> Attachments: external_attachments_alpha.patch
>
>
> For attachment-heavy applications storing the attachments in separate files
> significantly eases compaction problems.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
