[couchdb] branch 3.x updated: Fix splitting shards with large purge sequences
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch 3.x in repository https://gitbox.apache.org/repos/asf/couchdb.git The following commit(s) were added to refs/heads/3.x by this push: new 9f08191 Fix splitting shards with large purge sequences 9f08191 is described below commit 9f081914fe1fd7f31c2c1c7b3ead89427cf342f3 Author: Nick Vatamaniuc AuthorDate: Thu Sep 9 22:33:26 2021 -0400 Fix splitting shards with large purge sequences Previously, if the source db purge sequence > `purge_infos_limit`, shard splitting would crash with the `{{invalid_start_purge_seq,0}, [{couch_bt_engine,fold_purge_infos,5...` error. That was because purge sequences were always copied starting from 0. That would only work as long as the total number of purges stayed below the purge_infos_limit threshold. In order to correctly gather the purge sequences, the start sequence must be based off of the actual oldest sequence currently available. An example of how it should be done is in the `mem_rpc` module, when loading purge infos [0], so here we do exactly the same. The `MinSeq - 1` logic is also evident by inspecting the fold_purge_infos [1] function. The test sets up the exact scenario as described above: reduces the purge info limit to 10 then purges 20 documents. By purging more than the limit, we ensure the starting sequence is now != 0. However, the purge sequence btree is actually trimmed down during compaction. That is why there are a few extra helper functions to ensure compaction runs and finishes before shard splitting starts. Fixes: https://github.com/apache/couchdb/issues/3738 [0] https://github.com/apache/couchdb/blob/4ea9f1ea1a2078162d0e281948b56469228af3f7/src/mem3/src/mem3_rpc.erl#L206-L207 [1] https://github.com/apache/couchdb/blob/3.x/src/couch/src/couch_bt_engine.erl#L625-L637 --- src/couch/src/couch_db_split.erl | 3 +- src/mem3/test/eunit/mem3_reshard_test.erl | 81 +++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/couch/src/couch_db_split.erl b/src/couch/src/couch_db_split.erl index 3a1f98d..1aa86fb 100644 --- a/src/couch/src/couch_db_split.erl +++ b/src/couch/src/couch_db_split.erl @@ -301,7 +301,8 @@ copy_meta(#state{source_db = SourceDb, targets = Targets} = State) -> copy_purge_info(#state{source_db = Db} = State) -> -{ok, NewState} = couch_db:fold_purge_infos(Db, 0, fun purge_cb/2, State), +Seq = max(0, couch_db:get_oldest_purge_seq(Db) - 1), +{ok, NewState} = couch_db:fold_purge_infos(Db, Seq, fun purge_cb/2, State), Targets = maps:map(fun(_, #target{} = T) -> commit_purge_infos(T) end, NewState#state.targets), diff --git a/src/mem3/test/eunit/mem3_reshard_test.erl b/src/mem3/test/eunit/mem3_reshard_test.erl index 7cd6b1f..1122590 100644 --- a/src/mem3/test/eunit/mem3_reshard_test.erl +++ b/src/mem3/test/eunit/mem3_reshard_test.erl @@ -65,6 +65,7 @@ mem3_reshard_db_test_() -> fun setup/0, fun teardown/1, [ fun split_one_shard/1, +fun split_shard_with_lots_of_purges/1, fun update_docs_before_topoff1/1, fun indices_are_built/1, fun split_partitioned_db/1, @@ -140,6 +141,56 @@ split_one_shard(#{db1 := Db}) -> end)}. +% Test to check that shard with high number of purges can be split +split_shard_with_lots_of_purges(#{db1 := Db}) -> +{timeout, ?TIMEOUT, ?_test(begin +% Set a low purge infos limit, we are planning to overrun it +set_purge_infos_limit(Db, 10), + +% Add docs 1..20 and purge them +add_test_docs(Db, #{docs => [1, 20]}), +IdRevs = maps:fold(fun(Id, #{<<"_rev">> := Rev}, Acc) -> +[{Id, [Rev]} | Acc] +end, [], get_all_docs(Db)), +?assertMatch({ok, _}, purge_docs(Db, IdRevs)), + +% Compact to trim the purge sequence +ok = compact(Db), + +% Add some extra docs, these won't be purged +add_test_docs(Db, #{docs => [21, 30]}), +Docs0 = get_all_docs(Db), + +% Save db info before splitting +DbInfo0 = get_db_info(Db), + +% Split the one shard +[#shard{name=Shard}] = lists:sort(mem3:local_shards(Db)), +{ok, JobId} = mem3_reshard:start_split_job(Shard), +wait_state(JobId, completed), + +% Perform some basic checks that the shard was split +Shards1 = lists:sort(mem3:local_shards(Db)), +?assertEqual(2, length(Shards1)), +[#shard{range = R1}, #shard{range = R2}] = Shards1, +?assertEqual([16#, 16#7fff], R1), +?assertEqual([16#8000, 16#], R2), + +% Check metadata bits after the split +?assertEqual(10, get_purge_infos_limit(Db)), + +DbInfo1 =
[couchdb] branch fix-shard-splitting-with-lots-of-purges updated (ce2293d -> 9721046)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch fix-shard-splitting-with-lots-of-purges in repository https://gitbox.apache.org/repos/asf/couchdb.git. discard ce2293d Fix splitting shards with large purge sequences add 9721046 Fix splitting shards with large purge sequences This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (ce2293d) \ N -- N -- N refs/heads/fix-shard-splitting-with-lots-of-purges (9721046) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: src/mem3/test/eunit/mem3_reshard_test.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[couchdb] branch port-csp-patch-to-3.1.x updated (d2bd6df -> fc62f7e)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch port-csp-patch-to-3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git. discard d2bd6df feat: backport fine-grained CSP support to 3.1.x add fc62f7e feat: backport fine-grained CSP support to 3.1.x This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (d2bd6df) \ N -- N -- N refs/heads/port-csp-patch-to-3.1.x (fc62f7e) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: src/chttpd/test/eunit/chttpd_db_test.erl | 25 + 1 file changed, 25 insertions(+)
[couchdb-fauxton] branch main updated: Replace safeURLName usage (#1320)
This is an automated email from the ASF dual-hosted git repository. amaranhao pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/couchdb-fauxton.git The following commit(s) were added to refs/heads/main by this push: new 09e9f93 Replace safeURLName usage (#1320) 09e9f93 is described below commit 09e9f9353c766e249d2897c0b6512d01d8ff5998 Author: Deepak Chethan AuthorDate: Fri Sep 10 19:12:43 2021 +0530 Replace safeURLName usage (#1320) --- app/addons/documents/index-editor/components/IndexEditor.js | 4 +--- app/addons/documents/resources.js | 3 +-- app/addons/search/components/SearchIndexEditor.js | 3 +-- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/app/addons/documents/index-editor/components/IndexEditor.js b/app/addons/documents/index-editor/components/IndexEditor.js index b2a14bf..1ffafae 100644 --- a/app/addons/documents/index-editor/components/IndexEditor.js +++ b/app/addons/documents/index-editor/components/IndexEditor.js @@ -101,9 +101,7 @@ export default class IndexEditor extends Component { if (this.props.designDocId === 'new-doc' || this.props.isNewView) { return '#' + FauxtonAPI.urls('allDocs', 'app', encodedDatabase, encodedPartitionKey); } -const encodedDDoc = this.props.designDocId.startsWith('_design/') ? - '_design/' + encodeURIComponent(this.props.designDocId.substring(8)) : - encodeURIComponent(this.props.designDocId); +const encodedDDoc = app.utils.getSafeIdForDoc(this.props.designDocId); const encodedView = encodeURIComponent(this.props.viewName); return '#' + FauxtonAPI.urls('view', 'showView', encodedDatabase, encodedPartitionKey, encodedDDoc, encodedView); } diff --git a/app/addons/documents/resources.js b/app/addons/documents/resources.js index a920b8e..bb44518 100644 --- a/app/addons/documents/resources.js +++ b/app/addons/documents/resources.js @@ -64,8 +64,7 @@ Documents.DdocInfo = FauxtonAPI.Model.extend({ // treated separately. For instance, we could default into the // json editor for docs, or into a ddoc specific page. safeID: function () { -var ddoc = this.id.replace(/^_design\//, ""); -return "_design/" + app.utils.safeURLName(ddoc); +return app.utils.getSafeIdForDoc(this.id); } }); diff --git a/app/addons/search/components/SearchIndexEditor.js b/app/addons/search/components/SearchIndexEditor.js index dbff53a..825b656 100644 --- a/app/addons/search/components/SearchIndexEditor.js +++ b/app/addons/search/components/SearchIndexEditor.js @@ -96,8 +96,7 @@ export default class SearchIndexEditor extends React.Component { return '#' + FauxtonAPI.urls('allDocs', 'app', encodedDatabase, encodedPartitionKey); } -const cleanDDocName = this.props.lastSavedDesignDocName.replace(/^_design\//, ''); -const encodedDDoc = '_design/' + encodeURIComponent(cleanDDocName); +const encodedDDoc = app.utils.getSafeIdForDoc(this.props.lastSavedDesignDocName); const encodedIndex = encodeURIComponent(this.props.lastSavedSearchIndexName); return '#' + FauxtonAPI.urls('search', 'showIndex', encodedDatabase, encodedPartitionKey, encodedDDoc, encodedIndex);
[couchdb-nano] branch main updated (da5a5db -> 8552352)
This is an automated email from the ASF dual-hosted git repository. glynnbird pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/couchdb-nano.git. from da5a5db document custom certification authority in README add 8552352 9.0.4 No new revisions were added by this update. Summary of changes: package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
[couchdb] branch fix-shard-splitting-with-lots-of-purges updated (f51a949 -> ce2293d)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch fix-shard-splitting-with-lots-of-purges in repository https://gitbox.apache.org/repos/asf/couchdb.git. omit f51a949 Fix splitting shards with large purge sequences add ce2293d Fix splitting shards with large purge sequences This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (f51a949) \ N -- N -- N refs/heads/fix-shard-splitting-with-lots-of-purges (ce2293d) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
[couchdb] 01/01: Remove log line from CSP logic in chttpd_util
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch remove-log-line-from-csp-header-logic in repository https://gitbox.apache.org/repos/asf/couchdb.git commit adc2cc22ed499e2325bd44ca05eeff469d76cb1c Author: Nick Vatamaniuc AuthorDate: Fri Sep 10 00:58:20 2021 -0400 Remove log line from CSP logic in chttpd_util --- src/chttpd/src/chttpd_util.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/chttpd/src/chttpd_util.erl b/src/chttpd/src/chttpd_util.erl index 2640320..6c68568 100644 --- a/src/chttpd/src/chttpd_util.erl +++ b/src/chttpd/src/chttpd_util.erl @@ -65,7 +65,6 @@ get_chttpd_auth_config_boolean(Key, Default) -> maybe_add_csp_header(Component, OriginalHeaders, DefaultHeaderValue) -> Enabled = config:get_boolean("csp", Component ++ "_enable", true), -couch_log:notice("~n> maybe_add_csp_header~n Component: ~n~p~n OriginalHeaders: ~n~p~n DefaultHeaderValue: ~n~p~n, Enabled: ~n~p~n", [Component, OriginalHeaders, DefaultHeaderValue, Enabled]), case Enabled of true -> HeaderValue = config:get("csp", Component ++ "_header_value", DefaultHeaderValue),
[couchdb] branch remove-log-line-from-csp-header-logic created (now adc2cc2)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch remove-log-line-from-csp-header-logic in repository https://gitbox.apache.org/repos/asf/couchdb.git. at adc2cc2 Remove log line from CSP logic in chttpd_util This branch includes the following new commits: new adc2cc2 Remove log line from CSP logic in chttpd_util The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[couchdb] branch port-csp-patch-to-3.1.x updated (6d4822d -> d2bd6df)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch port-csp-patch-to-3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git. discard 6d4822d feat: backport fine-grained CSP support to 3.1.x add 8943d9c Reformat logfile-uploader.py to pass python format checker add d2bd6df feat: backport fine-grained CSP support to 3.1.x This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (6d4822d) \ N -- N -- N refs/heads/port-csp-patch-to-3.1.x (d2bd6df) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: build-aux/logfile-uploader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
[couchdb] branch 3.1.x updated: Reformat logfile-uploader.py to pass python format checker
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch 3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git The following commit(s) were added to refs/heads/3.1.x by this push: new 8943d9c Reformat logfile-uploader.py to pass python format checker 8943d9c is described below commit 8943d9c33b8cb31d451f2a3064dcd427a6b2a153 Author: Nick Vatamaniuc AuthorDate: Fri Sep 10 02:19:06 2021 -0400 Reformat logfile-uploader.py to pass python format checker --- build-aux/logfile-uploader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-aux/logfile-uploader.py b/build-aux/logfile-uploader.py index 3df9e6c..f33915a 100755 --- a/build-aux/logfile-uploader.py +++ b/build-aux/logfile-uploader.py @@ -34,7 +34,7 @@ def _tojson(req): def collect_logfiles(): -""" Find and tarball all logfiles """ +"""Find and tarball all logfiles""" tb = tarfile.open(name=TARFILE, mode="w:gz") # Test results for log in glob.glob("test-results.log"): @@ -52,7 +52,7 @@ def collect_logfiles(): def build_ci_doc(): -""" Build a metadata document with relevant detail from CI env """ +"""Build a metadata document with relevant detail from CI env""" doc = {} if "TRAVIS" in os.environ: doc["builder"] = "travis" @@ -125,7 +125,7 @@ def upload_logs(): def main(): -""" Find latest logfile and upload to Couch logfile db. """ +"""Find latest logfile and upload to Couch logfile db.""" print("Uploading logfiles...") collect_logfiles() req = upload_logs()
[couchdb] branch fix-shard-splitting-with-lots-of-purges updated (eb2af64 -> f51a949)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch fix-shard-splitting-with-lots-of-purges in repository https://gitbox.apache.org/repos/asf/couchdb.git. discard eb2af64 Fix splitting shards with large purge sequences add e6a3b4a Remove debug log line from attachments handler add bf92529 Remove log line from CSP logic in chttpd_util add f51a949 Fix splitting shards with large purge sequences This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (eb2af64) \ N -- N -- N refs/heads/fix-shard-splitting-with-lots-of-purges (f51a949) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: src/chttpd/src/chttpd_db.erl | 1 - src/chttpd/src/chttpd_util.erl | 1 - 2 files changed, 2 deletions(-)
[couchdb] branch 3.1.x created (now ce596c6)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch 3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git. at ce596c6 Prep for 3.1.1-RC2 (#3148) No new revisions were added by this update.
[couchdb] branch port-csp-patch-to-3.1.x created (now 6d4822d)
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a change to branch port-csp-patch-to-3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git. at 6d4822d feat: backport fine-grained CSP support to 3.1.x This branch includes the following new commits: new 6d4822d feat: backport fine-grained CSP support to 3.1.x The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[couchdb] 01/01: feat: backport fine-grained CSP support to 3.1.x
This is an automated email from the ASF dual-hosted git repository. vatamane pushed a commit to branch port-csp-patch-to-3.1.x in repository https://gitbox.apache.org/repos/asf/couchdb.git commit 6d4822d4932ed191c5ae4dfd0d8be6b601829a34 Author: Jan Lehnardt AuthorDate: Fri Jul 23 22:28:42 2021 +0200 feat: backport fine-grained CSP support to 3.1.x The original commit [0] was in 3.x branch This introduces CSP settings for attachments and show/list funs and streamlines the configuration with the existing Fauxton CSP options. Deprecates the old `[csp] enable` and `[csp] header_value` config options, but they are honoured going forward. They are replaced with `[csp] utils_enable` and `[csp] utils_header_value` respectively. The functionality and default values remain the same. In addition, these new config options are added, along with their default values: ``` [csp] attachments_enable = false attachments_header_value = sandbox showlist_enable = false showlist_header_value = sandbox ``` When enabled, these add `Content-Security-Policy` headers to all attachment requests and to all non-JSON show and all list function responses. Co-authored-by: Nick Vatamaniuc Co-authored-by: Robert Newson [0] https://github.com/apache/couchdb/commit/64281c0358e206a54e3b1386a7bc3b3e7c30547f --- rel/overlay/etc/default.ini| 10 +- src/chttpd/src/chttpd_db.erl | 3 +- src/chttpd/src/chttpd_external.erl | 3 +- src/chttpd/src/chttpd_misc.erl | 13 +- src/chttpd/src/chttpd_util.erl | 56 +++ src/chttpd/test/eunit/chttpd_csp_tests.erl | 253 + src/chttpd/test/eunit/chttpd_db_test.erl | 45 + src/couch_mrview/src/couch_mrview_show.erl | 4 +- 8 files changed, 342 insertions(+), 45 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 8010626..8cf470f 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -281,10 +281,14 @@ iterations = 10 ; iterations for password hashing ; Set the SameSite cookie property for the auth cookie. If empty, the SameSite property is not set. ; same_site = -; CSP (Content Security Policy) Support for _utils +; CSP (Content Security Policy) Support [csp] -enable = true -; header_value = default-src 'self'; img-src 'self'; font-src *; script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; +;utils_enable = true +;utils_header_value = default-src 'self'; img-src 'self'; font-src *; script-src 'self' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; +;attachments_enable = false +;attachments_header_value = sandbox +;showlist_enable = false +;showlist_header_value = sandbox [cors] credentials = false diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index b995460..a2e9315 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -1447,7 +1447,7 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa atom_to_list(Enc), couch_httpd:accepted_encodings(Req) ), -Headers = [ +Headers0 = [ {"ETag", Etag}, {"Cache-Control", "must-revalidate"}, {"Content-Type", binary_to_list(Type)} @@ -1464,6 +1464,7 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa _ -> [{"Accept-Ranges", "none"}] end, +Headers = chttpd_util:maybe_add_csp_header("attachments", Headers0, "sandbox"), Len = case {Enc, ReqAcceptsAttEnc} of {identity, _} -> % stored and served in identity form diff --git a/src/chttpd/src/chttpd_external.erl b/src/chttpd/src/chttpd_external.erl index 451d87d..70b5235 100644 --- a/src/chttpd/src/chttpd_external.erl +++ b/src/chttpd/src/chttpd_external.erl @@ -139,7 +139,8 @@ send_external_response(Req, Response) -> Headers1 = default_or_content_type(CType, Headers0), case Json of nil -> -chttpd:send_response(Req, Code, Headers1, Data); +Headers2 = chttpd_util:maybe_add_csp_header("showlist", Headers1, "sandbox"), +chttpd:send_response(Req, Code, Headers2, Data); Json -> chttpd:send_json(Req, Code, Headers1, Json) end. diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index 830fea3..f52210a 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -93,8 +93,9 @@ handle_utils_dir_req(#httpd{method='GET'}=Req, DocumentRoot) -> {_ActionKey, "/", RelativePath} -> % GET /_utils/path or GET /_utils/ CachingHeaders = [{"Cache-Control", "private, must-revalidate"}], -EnableCsp = config:get("csp", "enable", "false"), -Headers = maybe_add_csp_headers(CachingHeaders, EnableCsp), +