Hi all, I have three CouchDB servers A, B and C in separate distant locations. A and B where first, are currently at version 2.3.1 and they replicate each other very well. I introduced C with version 3.2.1 planning to upgrade the other two once I got my system to work well with it (updated permissions mostly).
Is replication between versions 3.2.1 and 2.3.1 supposed to work well? I am asking because my replications from C to A and B are working unreliably. Sometimes they are fine, other times they are slow and other times they stop. I know that slow replications could be due to a slow network, but I just saw some errors in the log and maybe you could give me some hints on how to make them work better. I could also revert C to 2.3.1, but would like to try to solve this first with the current version. This is from the CouchDB log on server C: [error] 2022-08-04T15:28:49.367168Z nonode@nohost <0.586.4> -------- Replication `b07c9df0efb112fedc10985db9d271ee+continuous+create_target` (`http://serverC.fakedomain.com:5984/jobhandler/` -> `http://serverA.fakedomain.com:5984/jobhandler/`) failed: {function_clause, [{couch_replicator_scheduler_job,handle_info, [{#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state, {rep, {"b07c9df0efb112fedc10985db9d271ee", "+continuous+create_target"}, {httpdb,http://serverC.fakedomain.com:5984/jobhandler/, [{<<"basic">>, {[{<<"username">>,<<"DelfinoG">>}, {<<"password">>,<<"somepassword">>}]}}], [{"Accept","application/json"}, {"User-Agent","CouchDB-Replicator/3.2.1"}], 30000, [{socket_options,[{keepalive,true},{nodelay,false}]}], 5,250,nil,20,nil,undefined,nil}, {httpdb,http://serverA.fakedomain.com:5984/jobhandler/, [{<<"basic">>, {[{<<"username">>,<<"DelfinoG">>}, {<<"password">>,<<"somepassword">>}]}}], [{"Accept","application/json"}, {"User-Agent","CouchDB-Replicator/3.2.1"}], 30000, [{socket_options,[{keepalive,true},{nodelay,false}]}], 5,250,nil,20,nil,undefined,nil}, [{checkpoint_interval,30000}, {connection_timeout,30000}, {continuous,true}, {create_target,true}, {http_connections,20}, {retries,5}, {socket_options,[{keepalive,true},{nodelay,false}]}, {use_checkpoints,true}, {worker_batch_size,500}, {worker_processes,4}], {user_ctx,<<"DelfinoG">>,[<<"_admin">>],undefined}, db,nil,<<"C_to_A_for_jobhandler">>, <<"shards/e0000000-ffffffff/_replicator.1634594445">>, {1659,626821,915992}, [{revisions_checked,69534602}, {missing_revisions_found,21557}, {docs_read,21557}, {docs_written,21557}, {changes_pending,0}, {doc_write_failures,0}, {checkpointed_source_seq, <<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}, {source_seq, <<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}, {through_seq, <<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}]}, http://serverC.fakedomain.com:5984/jobhandler/, http://serverA.fakedomain.com:5984/jobhandler/, {httpdb,http://serverC.fakedomain.com:5984/jobhandler/,[], [{"Accept","application/json"}, {"User-Agent","CouchDB-Replicator/3.2.1"}], 30000, [{socket_options,[{keepalive,true},{nodelay,false}]}], 5,250,<0.11968.4>,20,nil,undefined, {couch_replicator_auth_session,{<0.12064.4>,1,30000}}}, {httpdb,http://serverA.fakedomain.com:5984/jobhandler/,[], [{"Accept","application/json"}, {"User-Agent","CouchDB-Replicator/3.2.1"}], 30000, [{socket_options,[{keepalive,true},{nodelay,false}]}], 5,250,<0.12085.4>,20,nil,undefined, {couch_replicator_auth_session,{<0.13574.4>,1,30000}}}, [{[{<<"session_id">>,<<"9b1b28c12a03142bfacbce80e572f849">>}, {<<"start_time">>,<<"Thu, 04 Aug 2022 15:27:01 GMT">>}, {<<"end_time">>,<<"Thu, 04 Aug 2022 15:27:35 GMT">>}, {<<"start_last_seq">>, <<"2813222-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa6JlLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCG77lgQ1INDVMtUkyINcQBZEg8zBDm9AlgQywSjSzNzCyINSQBZEg9zBD2zh9gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiATYoxSA5ycgyjSSDFkAM2g82iOPJMYhBpkmJhgakGXQAYtB9SPjsqgAbZJxiaW5pSppBDyAGQcPofBHYIANzM8tEA0N0bVkAwgSwOw">>}, {<<"end_last_seq">>, <<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}, {<<"recorded_seq">>, <<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}, {<<"missing_checked">>,69534602}, {<<"missing_found">>,21557}, {<<"docs_read">>,21557}, {<<"docs_written">>,21557}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"d456a74fe970f6ce66a2ce528bf2ea2b">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Thu, 04 Aug 2022 11:38:06 GMT">>}, {<<"start_last_seq">>, <<"2811220-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq395LlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb5PBhqQaGqZapJgQa4gDyJB4mCHMaW_BhlgkGlmamVkQa0gCyJB6mCHsHYVgQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiDjYoxSA5ycgyjSSDFkAM2g82iOOxHcQg06REQwPSDDoAMeg-JHx2XgcbZJxiaW5pSppBDyAGQcPoXALYIANzM8tEA0N0bVkAiZawYQ">>}, {<<"end_last_seq">>, <<"2813222-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa6JlLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCG77lgQ1INDVMtUkyINcQBZEg8zBDm9AlgQywSjSzNzCyINSQBZEg9zBD2zh9gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiATYoxSA5ycgyjSSDFkAM2g82iOPJMYhBpkmJhgakGXQAYtB9SPjsqgAbZJxiaW5pSppBDyAGQcPofBHYIANzM8tEA0N0bVkAwgSwOw">>}, {<<"recorded_seq">>, <<"2813222-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa6JlLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCG77lgQ1INDVMtUkyINcQBZEg8zBDm9AlgQywSjSzNzCyINSQBZEg9zBD2zh9gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiATYoxSA5ycgyjSSDFkAM2g82iOPJMYhBpkmJhgakGXQAYtB9SPjsqgAbZJxiaW5pSppBDyAGQcPofBHYIANzM8tEA0N0bVkAwgSwOw">>}, {<<"missing_checked">>,69528356}, {<<"missing_found">>,21514}, {<<"docs_read">>,21514}, {<<"docs_written">>,21514}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"45e117dd2e651a93f432fb5725c00494">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Thu, 04 Aug 2022 07:19:56 GMT">>}, {<<"start_last_seq">>, <<"2810440-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq583FyjGbmlokmZobICuHocJSQpAMskeZgjDt3iwIamGhqkWKSbEGuIAMiQeZghzWhPYEItEI0szMwtiDUkAGVIPM4S9gx1siIm5kWmKEbEuyWMBkgwNQApoznyIl1jcwQalGCQnGVmmkWTQAohB-8EGcTxaBDHINCnR0IA0gw5ADLoPCZ-dtmCDjFMszS1NSTPoAcQgaBid0wUbZGBuZploYIiuLQsAxR2uVA">>}, {<<"end_last_seq">>, <<"2811220-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq395LlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb5PBhqQaGqZapJgQa4gDyJB4mCHMaW_BhlgkGlmamVkQa0gCyJB6mCHsHYVgQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiDjYoxSA5ycgyjSSDFkAM2g82iOOxHcQg06REQwPSDDoAMeg-JHx2XgcbZJxiaW5pSppBDyAGQcPoXALYIANzM8tEA0N0bVkAiZawYQ">>}, {<<"recorded_seq">>, <<"2811220-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq395LlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb5PBhqQaGqZapJgQa4gDyJB4mCHMaW_BhlgkGlmamVkQa0gCyJB6mCHsHYVgQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiDjYoxSA5ycgyjSSDFkAM2g82iOOxHcQg06REQwPSDDoAMeg-JHx2XgcbZJxiaW5pSppBDyAGQcPoXALYIANzM8tEA0N0bVkAiZawYQ">>}, {<<"missing_checked">>,69522825}, {<<"missing_found">>,21512}, {<<"docs_read">>,21512}, {<<"docs_written">>,21512}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"306e23aa74e149ee0c6f956c665736cf">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Thu, 04 Aug 2022 03:03:07 GMT">>}, {<<"start_last_seq">>, <<"2810001-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq29HLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb_ZgQ1INDVMtUkyINcQBZEg8zBDmNHewIRaJRpZmZhbEGpIAMqQeZgh7-0mwISbmRqYpRsS6JI8FSDI0ACmgOfMhXmJxAxuUYpCcZGSZRpJBCyAG7QcbxPHIC2KQaVKioQFpBh2AGHQfEj47XoENMk6xNLc0Jc2gBxCDoGF0TgBskIG5mWWigSG6tiwAh_yvmg">>}, {<<"end_last_seq">>, <<"2810440-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq583FyjGbmlokmZobICuHocJSQpAMskeZgjDt3iwIamGhqkWKSbEGuIAMiQeZghzWhPYEItEI0szMwtiDUkAGVIPM4S9gx1siIm5kWmKEbEuyWMBkgwNQApoznyIl1jcwQalGCQnGVmmkWTQAohB-8EGcTxaBDHINCnR0IA0gw5ADLoPCZ-dtmCDjFMszS1NSTPoAcQgaBid0wUbZGBuZploYIiuLQsAxR2uVA">>}, {<<"recorded_seq">>, <<"2810440-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq583FyjGbmlokmZobICuHocJSQpAMskeZgjDt3iwIamGhqkWKSbEGuIAMiQeZghzWhPYEItEI0szMwtiDUkAGVIPM4S9gx1siIm5kWmKEbEuyWMBkgwNQApoznyIl1jcwQalGCQnGVmmkWTQAohB-8EGcTxaBDHINCnR0IA0gw5ADLoPCZ-dtmCDjFMszS1NSTPoAcQgaBid0wUbZGBuZploYIiuLQsAxR2uVA">>}, {<<"missing_checked">>,69519984}, {<<"missing_found">>,21511}, {<<"docs_read">>,21511}, {<<"docs_written">>,21511}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"138ff4d49f04aa4bce01bc0cf244bbc9">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Thu, 04 Aug 2022 00:53:49 GMT">>}, {<<"start_last_seq">>, <<"2809701-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq682FyjGbmlokmZobICuHocJSQpAMskeZgjDN02wIamGhqkWKSbEGuIAMiQeZghzmjzYEItEI0szMwtiDUkAGVIPM4S9fSHYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEDzHINCnR0IA0gw5ADLoPCZ8d68AGGadYmluakmbQA4hB0DA6-wdskIG5mWWigSG6tiwAUeevbQ">>}, {<<"end_last_seq">>, <<"2810001-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq29HLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb_ZgQ1INDVMtUkyINcQBZEg8zBDmNHewIRaJRpZmZhbEGpIAMqQeZgh7-0mwISbmRqYpRsS6JI8FSDI0ACmgOfMhXmJxAxuUYpCcZGSZRpJBCyAG7QcbxPHIC2KQaVKioQFpBh2AGHQfEj47XoENMk6xNLc0Jc2gBxCDoGF0TgBskIG5mWWigSG6tiwAh_yvmg">>}, {<<"recorded_seq">>, <<"2810001-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq29HLlCM3dLQJM3Q2ABdPQ4TkhSAZJI9zBCGb_ZgQ1INDVMtUkyINcQBZEg8zBDmNHewIRaJRpZmZhbEGpIAMqQeZgh7-0mwISbmRqYpRsS6JI8FSDI0ACmgOfMhXmJxAxuUYpCcZGSZRpJBCyAG7QcbxPHIC2KQaVKioQFpBh2AGHQfEj47XoENMk6xNLc0Jc2gBxCDoGF0TgBskIG5mWWigSG6tiwAh_yvmg">>}, {<<"missing_checked">>,69516742}, {<<"missing_found">>,21510}, {<<"docs_read">>,21510}, {<<"docs_written">>,21510}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"4289854c5478dcc56cf9b669bfd9ea42">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Wed, 03 Aug 2022 23:14:43 GMT">>}, {<<"start_last_seq">>, <<"2809649-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq684FyjGbmlokmZobICuHocJSQpAMskeZgjDN1WwIamGhqkWKSbEGuIAMiQeZghzmgTYEItEI0szMwtiDUkAGVIPM4S9fTbYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEAjHINCnR0IA0gw5ADLoPCZ8dS8AGGadYmluakmbQA4hB0DA6-wNskIG5mWWigSG6tiwAF8KvOQ">>}, {<<"end_last_seq">>, <<"2809701-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq682FyjGbmlokmZobICuHocJSQpAMskeZgjDN02wIamGhqkWKSbEGuIAMiQeZghzmjzYEItEI0szMwtiDUkAGVIPM4S9fSHYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEDzHINCnR0IA0gw5ADLoPCZ8d68AGGadYmluakmbQA4hB0DA6-wdskIG5mWWigSG6tiwAUeevbQ">>}, {<<"recorded_seq">>, <<"2809701-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq682FyjGbmlokmZobICuHocJSQpAMskeZgjDN02wIamGhqkWKSbEGuIAMiQeZghzmjzYEItEI0szMwtiDUkAGVIPM4S9fSHYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEDzHINCnR0IA0gw5ADLoPCZ8d68AGGadYmluakmbQA4hB0DA6-wdskIG5mWWigSG6tiwAUeevbQ">>}, {<<"missing_checked">>,69511194}, {<<"missing_found">>,21509}, {<<"docs_read">>,21509}, {<<"docs_written">>,21509}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"dc9acce701b100bec794f2026479a371">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Wed, 03 Aug 2022 22:56:47 GMT">>}, {<<"start_last_seq">>, <<"2809620-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBqy83FyjGbmlokmZobICuHocJSQpAMskeZgjDN2WwIamGhqkWKSbEGuIAMiQeZghzmijYEItEI0szMwtiDUkAGVIPM4S9fTrYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4nj4D2KQaVKioQFpBh2AGHQfEj475oENMk6xNLc0Jc2gBxCDoGF09hvYIANzM8tEA0N0bVkAprCwGw">>}, {<<"end_last_seq">>, <<"2809649-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq684FyjGbmlokmZobICuHocJSQpAMskeZgjDN1WwIamGhqkWKSbEGuIAMiQeZghzmgTYEItEI0szMwtiDUkAGVIPM4S9fTbYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEAjHINCnR0IA0gw5ADLoPCZ8dS8AGGadYmluakmbQA4hB0DA6-wNskIG5mWWigSG6tiwAF8KvOQ">>}, {<<"recorded_seq">>, <<"2809649-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBq684FyjGbmlokmZobICuHocJSQpAMskeZgjDN1WwIamGhqkWKSbEGuIAMiQeZghzmgTYEItEI0szMwtiDUkAGVIPM4S9fTbYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4njEAjHINCnR0IA0gw5ADLoPCZ8dS8AGGadYmluakmbQA4hB0DA6-wNskIG5mWWigSG6tiwAF8KvOQ">>}, {<<"missing_checked">>,69508321}, {<<"missing_found">>,21508}, {<<"docs_read">>,21508}, {<<"docs_written">>,21508}, {<<"doc_write_failures">>,0}]}, {[{<<"session_id">>,<<"c382db252b544144265844ce0328fe29">>}, {<<"start_time">>,<<"Sun, 17 Jul 2022 01:19:22 GMT">>}, {<<"end_time">>,<<"Wed, 03 Aug 2022 22:47:27 GMT">>}, {<<"start_last_seq">>, <<"2809604-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBqy8rFyjGbmlokmZobICuHocJSQpAMskeZgjDNyWwIamGhqkWKSbEGuIAMiQeZghzmhDYEItEI0szMwtiDUkAGVIPM4S9fSrYEBNzI9MUI2JdkscCJBkagBTQnPkQL7G4gQ1KMUhOMrJMI8mgBRCD9oMN4nj4G2KQaVKioQFpBh2AGHQfEj47ZoMNMk6xNLc0Jc2gBxCDoGF09i... [error] 2022-08-04T15:28:49.367905Z nonode@nohost <0.19240.4> -------- Replicator, request GET to http://serverC.fakedomain.com:5984/jobhandler/_changes?feed=continuous&style=all_docs&since=%222815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA%22&timeout=10000 failed due to error closing_on_request [notice] 2022-08-04T15:28:49.368386Z nonode@nohost <0.19240.4> -------- Retrying _changes request to source database http://serverC.fakedomain.com:5984/jobhandler/ with since=<<"2815508-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9LuXKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYMzTAhlgkGlmamVkQa0gCyJB6mCHs3d1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZO8Qg06REQwPSDDoAMeg-JHx284ENMk6xNLc0Jc2gBxCDoGF0UQJskIG5mWWigSG6tiwAD8CvMw">> [error] 2022-08-04T15:28:49.368786Z nonode@nohost <0.586.4> -------- gen_server {couch_replicator_scheduler_job,{[98,48,55,99,57,100,102,48,101,102,98,49,49,50,102,101,100,99,49,48,57,56,53,100,98,57,100,50,55,49,101,101],[43,99,111,110,116,105,110,117,111,117,115,43,99,114,101,97,116,101,95,116,97,114,103,101,116]}} terminated with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) last msg: redacted state: [{rep_id,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"}},{source,http://serverC.fakedomain.com:5984/jobhandler/},{target,http://serverA.fakedomain.com:5984/jobhandler/},{db_name,<<"shards/e0000000-ffffffff/_replicator.1634594445">>},{doc_id,<<"C_to_A_for_jobhandler">>},{options,[{checkpoint_interval,30000},{connection_timeout,30000},{continuous,true},{create_target,true},{http_connections,20},{retries,5},{socket_options,[{keepalive,true},{nodelay,false}]},{use_checkpoints,true},{worker_batch_size,500},{worker_processes,4}]},{session_id,<<"29dd67dc40777f81e556c7a541c618e8">>},{start_seq,{0,<<"2815485-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9L2XKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYM9TBhlgkGlmamVkQa0gCyJB6mCHs3R1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZE8Qg06REQwPSDDoAMeg-JHx284INMk6xNLc0Jc2gBxCDoGF0kR9skIG5mWWigSG6tiwA_TuvHA">>}},{source_seq,<<"2815498-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjPlsQBJhgdA6j8QZGUwJzGwXxTOBYqxG5ibWSYaGKJrw2vQAYhB98EGMe_mBRtknGJpbmmaRpJBCyAG7QcbxPHMDWxQimlSoqEBaQY1QAyaDzaIgcULYpBBcpKRJbEGJSUAyaR6ePh0d4INMTE3Mk0xMiHWEAeQIfEwQ5gzNMCGWCQaWZqZWRBriALIEHuYIQw_3oMNSTU0TLVIIdYliQxJ8jATuCbtAptgaWiSZmhsgK4-CwCE1q8p">>},{committed_seq,{14,<<"2815507-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9LuXKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYMzTAhlgkGlmamVkQa0gCyJB6mCHs3V1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZO8Qg06REQwPSDDoAMeg-JHx284ENMk6xNLc0Jc2gBxCDoGF0UQJskIG5mWWigSG6tiwADn6vMg">>}},{current_through_seq,{14,<<"2815507-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9LuXKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYMzTAhlgkGlmamVkQa0gCyJB6mCHs3V1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZO8Qg06REQwPSDDoAMeg-JHx284ENMk6xNLc0Jc2gBxCDoGF0UQJskIG5mWWigSG6tiwADn6vMg">>}},{highest_seq_done,{14,<<"2815507-g1AAAAI7eJzLYWBg4MhgTmHgzcvPy09JdcjLz8gvLskBCjMlMiTJ____PyuDOYmBa9LuXKAYu6WhSZqhsQG6ehwmJCkAySR7mCEMP96DDUk1NEy1SDEh1hAHkCHxMEOYMzTAhlgkGlmamVkQa0gCyJB6mCHs3V1gQ0zMjUxTjIh1SR4LkGRoAFJAc-ZDvMTiBTYoxSA5ycgyjSSDFkAM2g82iOOZO8Qg06REQwPSDDoAMeg-JHx284ENMk6xNLc0Jc2gBxCDoGF0UQJskIG5mWWigSG6tiwADn6vMg">>}}] extra: [] [error] 2022-08-04T15:28:49.369309Z nonode@nohost <0.586.4> -------- CRASH REPORT Process (<0.586.4>) with 3 neighbors crashed with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247); initial_call: {couch_replicator_scheduler_job,init,['Argument__1']}, ancestors: [couch_replicator_scheduler_sup,couch_replicator_sup,...], message_queue_len: 2, links: [<0.19241.4>,<0.19244.4>,<0.19248.4>,<0.19246.4>,<0.19242.4>,...], dictionary: [{pending_count_state,{{1659,626929,27150},1}},{task_status_props,...},...], trap_exit: true, status: running, heap_size: 833026, stack_size: 27, reductions: 1466592 [error] 2022-08-04T15:28:49.369634Z nonode@nohost <0.403.0> -------- Supervisor couch_replicator_scheduler_sup had child undefined started with {couch_replicator_scheduler_job,start_link,undefined} at <0.586.4> exit with reason no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) in context child_terminated [notice] 2022-08-04T15:28:49.370104Z nonode@nohost <0.25870.8> 9502aa8b47 couchdb:5984 172.17.0.5 DelfinoG POST /jobhandlersims/_revs_diff 200 ok 2 [error] 2022-08-04T15:28:49.371112Z nonode@nohost <0.19242.4> -------- gen_server <0.19242.4> terminated with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) last msg: redacted state: [{main_pid,<0.586.4>},{loop,<0.19243.4>},{source,http://serverC.fakedomain.com:5984/jobhandler/},{target,http://serverA.fakedomain.com:5984/jobhandler/},{num_readers,0},{pending_fetch,nil},{batch_size,0}] extra: [] [error] 2022-08-04T15:28:49.371369Z nonode@nohost <0.19242.4> -------- CRASH REPORT Process (<0.19242.4>) with 0 neighbors exited with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) at gen_server:decode_msg/9(line:410) <= proc_lib:init_p_do_apply/3(line:247); initial_call: {couch_replicator_worker,init,['Argument__1']}, ancestors: [<0.586.4>,couch_replicator_scheduler_sup,couch_replicator_sup,...], message_queue_len: 1, links: [], dictionary: [{last_stats_report,{1659,626916,507736}}], trap_exit: true, status: running, heap_size: 46422, stack_size: 27, reductions: 674 [error] 2022-08-04T15:28:49.372531Z nonode@nohost <0.19246.4> -------- gen_server <0.19246.4> terminated with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) last msg: redacted state: [{main_pid,<0.586.4>},{loop,<0.19247.4>},{source,http://serverC.fakedomain.com:5984/jobhandler/},{target,http://serverA.fakedomain.com:5984/jobhandler/},{num_readers,0},{pending_fetch,nil},{batch_size,0}] extra: [] [error] 2022-08-04T15:28:49.374095Z nonode@nohost <0.19248.4> -------- gen_server <0.19248.4> terminated with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) last msg: redacted state: [{main_pid,<0.586.4>},{loop,<0.19249.4>},{source,http://serverC.fakedomain.com:5984/jobhandler/},{target,http://serverA.fakedomain.com:5984/jobhandler/},{num_readers,0},{pending_fetch,nil},{batch_size,0}] extra: [] [error] 2022-08-04T15:28:49.374496Z nonode@nohost <0.19246.4> -------- CRASH REPORT Process (<0.19246.4>) with 1 neighbors exited with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) at gen_server:decode_msg/9(line:410) <= proc_lib:init_p_do_apply/3(line:247); initial_call: {couch_replicator_worker,init,['Argument__1']}, ancestors: [<0.586.4>,couch_replicator_scheduler_sup,couch_replicator_sup,...], message_queue_len: 0, links: [<0.19247.4>], dictionary: [{last_stats_report,{1659,626914,533399}}], trap_exit: true, status: running, heap_size: 46422, stack_size: 27, reductions: 600 [error] 2022-08-04T15:28:49.375406Z nonode@nohost <0.19248.4> -------- CRASH REPORT Process (<0.19248.4>) with 0 neighbors exited with reason: no function clause matching couch_replicator_scheduler_job:handle_info({#Ref<0.3396698094.4198236163.42579>,{error,closing_on_request}}, {rep_state,{rep,{"b07c9df0efb112fedc10985db9d271ee","+continuous+create_target"},{httpdb,"http:/...",...},...},...})(line:239) at gen_server:try_dispatch/4(line:616) <= gen_server:handle_msg/6(line:686) <= proc_lib:init_p_do_apply/3(line:247) at gen_server:decode_msg/9(line:410) <= proc_lib:init_p_do_apply/3(line:247); initial_call: {couch_replicator_worker,init,['Argument__1']}, ancestors: [<0.586.4>,couch_replicator_scheduler_sup,couch_replicator_sup,...], message_queue_len: 1, links: [], dictionary: [{last_stats_report,{1659,626923,16597}}], trap_exit: true, status: running, heap_size: 46422, stack_size: 27, reductions: 674 [notice] 2022-08-04T15:28:49.405268Z nonode@nohost <0.27447.8> 29fe2c5189 couchdb:5984 172.17.0.6 DelfinoG GET /jobhandlersims/20220804160447010DUCHOWSKIM 200 ok 4 [info] 2022-08-04T15:28:49.410896Z nonode@nohost <0.25683.8> -------- Starting index update for db: shards/c0000000-dfffffff/jobhandlersims.1630631349 idx: _design/as Regards, Gustavo Delfino
