Hello,
I setup a new 3-node PostgreSQL cluster with HA managed by PAF. Nodes are
named ph-sql-03, ph-sql-04, ph-sql-05. Archive mode is on and writing
archive files to an NFS share that's mounted on all nodes using pgBackRest.
What I did:
- Create a pacemaker cluster, cib.xml is attached.
- Set maintenance-mode=true in pacemaker
- Bring up ph-sql-03 with pg_ctl start
- Take a pg_basebackup on ph-sql-04 and ph-sql-05
- Create a recovery.conf on ph-sql-04 and ph-sql-05:
standby_mode = 'on'
primary_conninfo = 'user=replication password=XXXXXXXXXXXXXXXX
application_name=ph-sql-0x host=10.100.130.20 port=5432 sslmode=prefer
sslcompression=0 krbsrvname=postgres target_session_attrs=any'
recovery_target_timeline = 'latest'
restore_command = 'pgbackrest --stanza=pgdb2 archive-get %f "%p"'
- Bring up ph-sql-04 and ph-sql-05 and let recovery finish
- Set maintenance-mode=false in pacemaker
- Cluster is now running with ph-sql-03 as master and ph-sql-04/5 as slaves
At this point I tried a manual failover:
- pcs resource move --wait --master pgsql-ha ph-sql-04
Contrary to my expectations, pacemaker attempted to stop psqld on
ph-sql-03. This took longer than the configured timeout of 60s (checkpoint
hadn't completed yet) and the node was fenced. Then I ended up with
ph-sql-04 and ph-sql-05 both in slave mode and ph-sql-03 rebooting.
Master: pgsql-ha
Meta Attrs: notify=true
Resource: pgsqld (class=ocf provider=heartbeat type=pgsqlms)
Attributes: bindir=/usr/pgsql-11/bin pgdata=/var/lib/pgsql/11/data
recovery_template=/var/lib/pgsql/recovery.conf.pcmk
Operations: demote interval=0s timeout=30s (pgsqld-demote-interval-0s)
methods interval=0s timeout=5 (pgsqld-methods-interval-0s)
monitor interval=15s role=Master timeout=10s
(pgsqld-monitor-interval-15s)
monitor interval=16s role=Slave timeout=10s
(pgsqld-monitor-interval-16s)
notify interval=0s timeout=60s (pgsqld-notify-interval-0s)
promote interval=0s timeout=30s (pgsqld-promote-interval-0s)
reload interval=0s timeout=20 (pgsqld-reload-interval-0s)
start interval=0s timeout=60s (pgsqld-start-interval-0s)
stop interval=0s timeout=60s (pgsqld-stop-interval-0s)
I understand I should at least increase the timeout of the stop operation
for psqld, though I'm not sure how much. Checkpoints can take up to 15
minutes to complete on this cluster. So is 20 minutes reasonable? Any other
operations I should increase the timeouts for?
Why didn't pacemaker elect and promote one of the other nodes?
--
Tiemen Ruiten
Infrastructure Engineer
R&D Media
<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.10" epoch="323" num_updates="0" admin_epoch="0" cib-last-written="Fri Jun 14 09:55:24 2019" update-origin="ph-sql-04" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3">
<configuration>
<crm_config>
<cluster_property_set id="cib-bootstrap-options">
<nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.19-8.el7_6.4-c3c624ea3d"/>
<nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
<nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="pgdb2"/>
<nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1559847799"/>
<nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="true"/>
</cluster_property_set>
</crm_config>
<nodes>
<node id="1" uname="ph-sql-03">
<instance_attributes id="nodes-1">
<nvpair id="nodes-1-master-pgsqld" name="master-pgsqld" value="1001"/>
</instance_attributes>
</node>
<node id="2" uname="ph-sql-04">
<instance_attributes id="nodes-2">
<nvpair id="nodes-2-master-pgsqld" name="master-pgsqld" value="1000"/>
</instance_attributes>
</node>
<node id="3" uname="ph-sql-05">
<instance_attributes id="nodes-3">
<nvpair id="nodes-3-master-pgsqld" name="master-pgsqld" value="990"/>
</instance_attributes>
</node>
</nodes>
<resources>
<primitive class="stonith" id="fence_ph_sql_03" type="fence_ipmilan">
<instance_attributes id="fence_ph_sql_03-instance_attributes">
<nvpair id="fence_ph_sql_03-instance_attributes-ipaddr" name="ipaddr" value="10.100.4.113"/>
<nvpair id="fence_ph_sql_03-instance_attributes-login" name="login" value="stonith"/>
<nvpair id="fence_ph_sql_03-instance_attributes-passwd" name="passwd" value="XXXXXXXXXXXXXXX"/>
<nvpair id="fence_ph_sql_03-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="ph-sql-03"/>
</instance_attributes>
<operations>
<op id="fence_ph_sql_03-monitor-interval-60s" interval="60s" name="monitor"/>
</operations>
</primitive>
<primitive class="stonith" id="fence_ph_sql_04" type="fence_ipmilan">
<instance_attributes id="fence_ph_sql_04-instance_attributes">
<nvpair id="fence_ph_sql_04-instance_attributes-ipaddr" name="ipaddr" value="10.100.4.114"/>
<nvpair id="fence_ph_sql_04-instance_attributes-login" name="login" value="stonith"/>
<nvpair id="fence_ph_sql_04-instance_attributes-passwd" name="passwd" value="XXXXXXXXXXXXXXX"/>
<nvpair id="fence_ph_sql_04-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="ph-sql-04"/>
</instance_attributes>
<operations>
<op id="fence_ph_sql_04-monitor-interval-60s" interval="60s" name="monitor"/>
</operations>
</primitive>
<primitive class="stonith" id="fence_ph_sql_05" type="fence_ipmilan">
<instance_attributes id="fence_ph_sql_05-instance_attributes">
<nvpair id="fence_ph_sql_05-instance_attributes-ipaddr" name="ipaddr" value="10.100.4.115"/>
<nvpair id="fence_ph_sql_05-instance_attributes-login" name="login" value="stonith"/>
<nvpair id="fence_ph_sql_05-instance_attributes-passwd" name="passwd" value="XXXXXXXXXXXXXXX"/>
<nvpair id="fence_ph_sql_05-instance_attributes-pcmk_host_list" name="pcmk_host_list" value="ph-sql-05"/>
</instance_attributes>
<operations>
<op id="fence_ph_sql_05-monitor-interval-60s" interval="60s" name="monitor"/>
</operations>
</primitive>
<master id="pgsql-ha">
<primitive class="ocf" id="pgsqld" provider="heartbeat" type="pgsqlms">
<instance_attributes id="pgsqld-instance_attributes">
<nvpair id="pgsqld-instance_attributes-bindir" name="bindir" value="/usr/pgsql-11/bin"/>
<nvpair id="pgsqld-instance_attributes-pgdata" name="pgdata" value="/var/lib/pgsql/11/data"/>
<nvpair id="pgsqld-instance_attributes-recovery_template" name="recovery_template" value="/var/lib/pgsql/recovery.conf.pcmk"/>
</instance_attributes>
<operations>
<op id="pgsqld-demote-interval-0s" interval="0s" name="demote" timeout="1200s"/>
<op id="pgsqld-methods-interval-0s" interval="0s" name="methods" timeout="5"/>
<op id="pgsqld-monitor-interval-15s" interval="15s" name="monitor" role="Master" timeout="10s"/>
<op id="pgsqld-monitor-interval-16s" interval="16s" name="monitor" role="Slave" timeout="10s"/>
<op id="pgsqld-notify-interval-0s" interval="0s" name="notify" timeout="60s"/>
<op id="pgsqld-promote-interval-0s" interval="0s" name="promote" timeout="30s"/>
<op id="pgsqld-reload-interval-0s" interval="0s" name="reload" timeout="20"/>
<op id="pgsqld-start-interval-0s" interval="0s" name="start" timeout="60s"/>
<op id="pgsqld-stop-interval-0s" interval="0s" name="stop" timeout="1200s"/>
</operations>
</primitive>
<meta_attributes id="pgsql-ha-meta_attributes">
<nvpair id="pgsql-ha-meta_attributes-notify" name="notify" value="true"/>
</meta_attributes>
</master>
<primitive class="ocf" id="pgsql-master-ip" provider="heartbeat" type="IPaddr2">
<instance_attributes id="pgsql-master-ip-instance_attributes">
<nvpair id="pgsql-master-ip-instance_attributes-cidr_netmask" name="cidr_netmask" value="24"/>
<nvpair id="pgsql-master-ip-instance_attributes-ip" name="ip" value="10.100.130.20"/>
</instance_attributes>
<operations>
<op id="pgsql-master-ip-monitor-interval-10s" interval="10s" name="monitor"/>
<op id="pgsql-master-ip-start-interval-0s" interval="0s" name="start" timeout="20s"/>
<op id="pgsql-master-ip-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
</operations>
</primitive>
<primitive class="systemd" id="pgsql-backupfull" type="[email protected]">
<operations>
<op id="pgsql-backupfull-monitor-interval-60s" interval="60s" name="monitor"/>
<op id="pgsql-backupfull-start-interval-0s" interval="0s" name="start" timeout="100"/>
<op id="pgsql-backupfull-stop-interval-0s" interval="0s" name="stop" timeout="100"/>
</operations>
</primitive>
<primitive class="systemd" id="pgsql-backupincr" type="[email protected]">
<operations>
<op id="pgsql-backupincr-monitor-interval-60s" interval="60s" name="monitor"/>
<op id="pgsql-backupincr-start-interval-0s" interval="0s" name="start" timeout="100"/>
<op id="pgsql-backupincr-stop-interval-0s" interval="0s" name="stop" timeout="100"/>
</operations>
</primitive>
</resources>
<constraints>
<rsc_location id="location-fence_ph_sql_05-ph-sql-05--INFINITY" node="ph-sql-05" rsc="fence_ph_sql_05" score="-INFINITY"/>
<rsc_location id="location-fence_ph_sql_04-ph-sql-04--INFINITY" node="ph-sql-04" rsc="fence_ph_sql_04" score="-INFINITY"/>
<rsc_location id="location-fence_ph_sql_03-ph-sql-03--INFINITY" node="ph-sql-03" rsc="fence_ph_sql_03" score="-INFINITY"/>
<rsc_colocation id="colocation-pgsql-master-ip-pgsql-ha-INFINITY" rsc="pgsql-master-ip" rsc-role="Started" score="INFINITY" with-rsc="pgsql-ha" with-rsc-role="Master"/>
<rsc_order first="pgsql-ha" first-action="promote" id="order-pgsql-ha-pgsql-master-ip-Mandatory" kind="Mandatory" symmetrical="false" then="pgsql-master-ip" then-action="start"/>
<rsc_order first="pgsql-ha" first-action="demote" id="order-pgsql-ha-pgsql-master-ip-Mandatory-1" kind="Mandatory" symmetrical="false" then="pgsql-master-ip" then-action="stop"/>
<rsc_colocation id="colocation-pgsql-backupfull-pgsql-ha-INFINITY" rsc="pgsql-backupfull" rsc-role="Started" score="INFINITY" with-rsc="pgsql-ha" with-rsc-role="Slave"/>
<rsc_colocation id="colocation-pgsql-backupincr-pgsql-ha-INFINITY" rsc="pgsql-backupincr" rsc-role="Started" score="INFINITY" with-rsc="pgsql-ha" with-rsc-role="Slave"/>
<rsc_order first="pgsql-ha" first-action="promote" id="order-pgsql-ha-pgsql-backupfull-mandatory" then="pgsql-backupfull" then-action="start"/>
<rsc_order first="pgsql-ha" first-action="promote" id="order-pgsql-ha-pgsql-backupincr-mandatory" then="pgsql-backupincr" then-action="start"/>
<rsc_location id="cli-prefer-pgsql-ha" node="ph-sql-04" role="Master" rsc="pgsql-ha" score="INFINITY"/>
</constraints>
<rsc_defaults>
<meta_attributes id="rsc_defaults-options">
<nvpair id="rsc_defaults-options-migration-threshold" name="migration-threshold" value="5"/>
<nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="10"/>
</meta_attributes>
</rsc_defaults>
</configuration>
<status>
<node_state id="2" uname="ph-sql-04" in_ccm="true" crmd="online" crm-debug-origin="do_state_transition" join="member" expected="member">
<lrm id="2">
<lrm_resources>
<lrm_resource id="fence_ph_sql_05" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_05_last_0" operation_key="fence_ph_sql_05_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="13:803:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;13:803:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="49" rc-code="0" op-status="0" interval="0" last-run="1560498856" last-rc-change="1560498856" exec-time="140" queue-time="0" op-digest="ed9455f0925a6450aeaa6e3d79b4f671" op-secure-params=" password passwd " op-secure-digest="babd733dfc85a73febf7ab10b566ceb0"/>
</lrm_resource>
<lrm_resource id="pgsqld" type="pgsqlms" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsqld_last_0" operation_key="pgsqld_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="16:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;16:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="34" rc-code="0" op-status="0" interval="0" last-run="1560498448" last-rc-change="1560498448" exec-time="207" queue-time="0" op-digest="acadac452337bb8de0485e48ffa05536" op-force-restart=" recovery_template datadir pgdata " op-restart-digest="ada54c00508744546e115cd461af2297"/>
</lrm_resource>
<lrm_resource id="pgsql-backupincr" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupincr_last_0" operation_key="pgsql-backupincr_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="56:801:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;56:801:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="44" rc-code="0" op-status="0" interval="0" last-run="1560498703" last-rc-change="1560498703" exec-time="2241" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="pgsql-backupfull" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupfull_last_0" operation_key="pgsql-backupfull_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="53:801:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;53:801:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="42" rc-code="0" op-status="0" interval="0" last-run="1560498703" last-rc-change="1560498703" exec-time="2146" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="pgsql-master-ip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsql-master-ip_last_0" operation_key="pgsql-master-ip_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="5:512:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;5:512:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="22" rc-code="7" op-status="0" interval="0" last-run="1560247978" last-rc-change="1560247978" exec-time="79" queue-time="0" op-digest="4d402315e2ed28a2e8f3debe69199080"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_03" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_03_last_0" operation_key="fence_ph_sql_03_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="1:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;1:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1560498448" last-rc-change="1560498448" exec-time="196" queue-time="0" op-digest="ed5ca13a85b29aac43b187bf3043fb68" op-secure-params=" password passwd " op-secure-digest="696c8fcb19743457de1067cb3dce4776"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_04" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_04_last_0" operation_key="fence_ph_sql_04_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="2:512:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;2:512:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-04" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1560247978" last-rc-change="1560247978" exec-time="0" queue-time="0" op-digest="8c6dd649e7d6cc1260f11123832d063a" op-secure-params=" password passwd " op-secure-digest="117cf5962eb7aa568d71a0238d0eddd6"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
<node_state id="1" uname="ph-sql-03" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
<lrm id="1">
<lrm_resources>
<lrm_resource id="fence_ph_sql_03" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_03_last_0" operation_key="fence_ph_sql_03_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="1:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;1:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="2" queue-time="0" op-digest="ed5ca13a85b29aac43b187bf3043fb68" op-secure-params=" password passwd " op-secure-digest="696c8fcb19743457de1067cb3dce4776"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_04" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_04_last_0" operation_key="fence_ph_sql_04_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="2:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;2:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="0" queue-time="0" op-digest="8c6dd649e7d6cc1260f11123832d063a" op-secure-params=" password passwd " op-secure-digest="117cf5962eb7aa568d71a0238d0eddd6"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_05" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_05_last_0" operation_key="fence_ph_sql_05_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="3:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;3:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="0" queue-time="0" op-digest="ed9455f0925a6450aeaa6e3d79b4f671" op-secure-params=" password passwd " op-secure-digest="babd733dfc85a73febf7ab10b566ceb0"/>
</lrm_resource>
<lrm_resource id="pgsqld" type="pgsqlms" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsqld_last_0" operation_key="pgsqld_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="4:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;4:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="18" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="166" queue-time="0" op-digest="acadac452337bb8de0485e48ffa05536" op-force-restart=" recovery_template datadir pgdata " op-restart-digest="ada54c00508744546e115cd461af2297"/>
</lrm_resource>
<lrm_resource id="pgsql-master-ip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsql-master-ip_last_0" operation_key="pgsql-master-ip_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="5:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;5:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="22" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="71" queue-time="0" op-digest="4d402315e2ed28a2e8f3debe69199080"/>
</lrm_resource>
<lrm_resource id="pgsql-backupfull" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupfull_last_0" operation_key="pgsql-backupfull_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;6:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="26" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="pgsql-backupincr" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupincr_last_0" operation_key="pgsql-backupincr_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="7:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" transition-magic="0:7;7:3:7:6c94a75c-9cbb-4976-b02a-293c642848ad" exit-reason="" on_node="ph-sql-03" call-id="30" rc-code="7" op-status="0" interval="0" last-run="1560499385" last-rc-change="1560499385" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
<node_state id="3" uname="ph-sql-05" crmd="online" crm-debug-origin="do_state_transition" in_ccm="true" join="member" expected="member">
<lrm id="3">
<lrm_resources>
<lrm_resource id="fence_ph_sql_05" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_05_last_0" operation_key="fence_ph_sql_05_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="3:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;3:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1560249061" last-rc-change="1560249061" exec-time="0" queue-time="0" op-digest="ed9455f0925a6450aeaa6e3d79b4f671" op-secure-params=" password passwd " op-secure-digest="babd733dfc85a73febf7ab10b566ceb0"/>
</lrm_resource>
<lrm_resource id="pgsqld" type="pgsqlms" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsqld_last_0" operation_key="pgsqld_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="14:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;14:792:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="31" rc-code="0" op-status="0" interval="0" last-run="1560498448" last-rc-change="1560498448" exec-time="198" queue-time="0" op-digest="acadac452337bb8de0485e48ffa05536" op-force-restart=" recovery_template datadir pgdata " op-restart-digest="ada54c00508744546e115cd461af2297"/>
</lrm_resource>
<lrm_resource id="pgsql-backupincr" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupincr_last_0" operation_key="pgsql-backupincr_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="7:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;7:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="30" rc-code="7" op-status="0" interval="0" last-run="1560249061" last-rc-change="1560249061" exec-time="11" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="pgsql-backupfull" type="[email protected]" class="systemd">
<lrm_rsc_op id="pgsql-backupfull_last_0" operation_key="pgsql-backupfull_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="6:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;6:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="26" rc-code="7" op-status="0" interval="0" last-run="1560249061" last-rc-change="1560249061" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
<lrm_resource id="pgsql-master-ip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="pgsql-master-ip_last_0" operation_key="pgsql-master-ip_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="5:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;5:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="22" rc-code="7" op-status="0" interval="0" last-run="1560249061" last-rc-change="1560249061" exec-time="77" queue-time="1" op-digest="4d402315e2ed28a2e8f3debe69199080"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_03" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_03_last_0" operation_key="fence_ph_sql_03_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="1:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:7;1:514:7:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1560249061" last-rc-change="1560249061" exec-time="2" queue-time="0" op-digest="ed5ca13a85b29aac43b187bf3043fb68" op-secure-params=" password passwd " op-secure-digest="696c8fcb19743457de1067cb3dce4776"/>
</lrm_resource>
<lrm_resource id="fence_ph_sql_04" type="fence_ipmilan" class="stonith">
<lrm_rsc_op id="fence_ph_sql_04_last_0" operation_key="fence_ph_sql_04_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="10:803:0:864a31dc-342c-47d4-a0ff-700f44fb6690" transition-magic="0:0;10:803:0:864a31dc-342c-47d4-a0ff-700f44fb6690" exit-reason="" on_node="ph-sql-05" call-id="38" rc-code="0" op-status="0" interval="0" last-run="1560498856" last-rc-change="1560498856" exec-time="130" queue-time="0" op-digest="8c6dd649e7d6cc1260f11123832d063a" op-secure-params=" password passwd " op-secure-digest="117cf5962eb7aa568d71a0238d0eddd6"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
</status>
</cib>
_______________________________________________
Manage your subscription:
https://lists.clusterlabs.org/mailman/listinfo/users
ClusterLabs home: https://www.clusterlabs.org/
