Hi,
that's my first attempt to mount a HA cluster. I have a strange behavior on my
drbd disks.
When I failover the cluster the DRBD Master node move correctly but the Slave
one doesn't start.
For simulate the fail over I change the weight of the preferred master node.
As you could see on my config I have 2 drbd disk and both of them has the same
problem.
Could you give me some suggestions? The log doesn't seem really show errors.
Thanks really much.
Here are my configuration file:
---------------
[r...@san-bal1 mnt]# cat /etc/drbd.conf
#
# drbd.conf example
#
skip {
As you can see, you can also comment chunks of text
with a 'skip[optional nonsense]{ skipped text }' section.
This comes in handy, if you just want to comment out
some 'resource <some name> {...}' section:
just precede it with 'skip'.
The basic format of option assignment is
<option name><linear whitespace><value>;
It should be obvious from the examples below,
but if you really care to know the details:
<option name> :=
valid options in the respective scope
<value> := <num>|<string>|<choice>|...
depending on the set of allowed values
for the respective option.
<num> := [0-9]+, sometimes with an optional suffix of K,M,G
<string> := (<name>|\"([^\"\\\n]*|\\.)*\")+
<name> := [/_.A-Za-z0-9-]+
}
#
# At most ONE global section is allowed.
# It must precede any resource section.
#
global {
usage-count yes;
}
common {
syncer { rate 40M; }
}
resource drbd0 {
protocol C;
handlers {
pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f";
pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt -f";
local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
outdate-peer "/usr/lib/heartbeat/drbd-peer-outdater -t 5";
}
startup {
degr-wfc-timeout 120; # 2 minutes.
}
disk {
on-io-error detach;
}
net {
after-sb-0pri disconnect;
after-sb-1pri disconnect;
after-sb-2pri disconnect;
rr-conflict disconnect;
}
syncer {
rate 40M;
al-extents 257;
}
on SAN-bal1 {
device /dev/drbd0;
disk /dev/cciss/c0d1p1;
address 192.168.0.1:7788;
meta-disk internal;
}
on SAN-bal2 {
device /dev/drbd0;
disk /dev/cciss/c0d1p1;
address 192.168.0.2:7788;
meta-disk internal;
}
}
resource drbd1 {
protocol C;
handlers {
pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f";
pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt -f";
local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
outdate-peer "/usr/lib/heartbeat/drbd-peer-outdater -t 5";
}
startup {
degr-wfc-timeout 120; # 2 minutes.
}
disk {
on-io-error detach;
}
net {
after-sb-0pri disconnect;
after-sb-1pri disconnect;
after-sb-2pri disconnect;
rr-conflict disconnect;
}
syncer {
rate 40M;
al-extents 257;
}
on SAN-bal1 {
device /dev/drbd1;
disk /dev/cciss/c0d1p2;
address 192.168.0.1:7789;
meta-disk internal;
}
on SAN-bal2 {
device /dev/drbd1;
disk /dev/cciss/c0d1p2;
address 192.168.0.2:7789;
meta-disk internal;
}
}
--------------------------------------------
[r...@san-bal1 mnt]# cat /var/lib/heartbeat/crm/cib.xml
<cib admin_epoch="0" have_quorum="true" ignore_dtd="false" num_peers="2"
cib_feature_revision="2.0" ccm_transition="2" generated="true"
dc_uuid="2a16d3bc-1774-43f7-8728-2410f82e8d32" epoch="296" num_updates="2"
cib-last-written="Wed Jan 21 13:24:53 2009">
<configuration>
<crm_config>
<cluster_property_set id="cib-bootstrap-options">
<attributes>
<nvpair id="cib-bootstrap-options-dc-version" name="dc-version"
value="2.1.3-node: 552305612591183b1628baa5bc6e903e0f1e26a3"/>
<nvpair name="last-lrm-refresh"
id="cib-bootstrap-options-last-lrm-refresh" value="1232536647"/>
</attributes>
</cluster_property_set>
</crm_config>
<nodes>
<node id="2a16d3bc-1774-43f7-8728-2410f82e8d32" uname="san-bal2"
type="normal"/>
<node id="ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7" uname="san-bal1"
type="normal"/>
</nodes>
<resources>
<master_slave id="ms-drbd0">
<meta_attributes id="ma-ms-drbd0">
<attributes>
<nvpair id="ma-ms-drbd0-1" name="clone_max" value="2"/>
<nvpair id="ma-ms-drbd0-2" name="clone_node_max" value="1"/>
<nvpair id="ma-ms-drbd0-3" name="master_max" value="1"/>
<nvpair id="ma-ms-drbd0-4" name="master_node_max" value="1"/>
<nvpair id="ma-ms-drbd0-5" name="notify" value="yes"/>
<nvpair id="ma-ms-drbd0-6" name="globally_unique" value="false"/>
<nvpair id="ma-ms-drbd0-7" name="target_role" value="started"/>
</attributes>
</meta_attributes>
<primitive id="drbd0" class="ocf" provider="heartbeat" type="drbd">
<instance_attributes id="ia-drbd0">
<attributes>
<nvpair id="ia-drbd0-1" name="drbd_resource" value="drbd0"/>
</attributes>
</instance_attributes>
<operations>
<op id="op-drbd0-1" name="monitor" interval="59s" timeout="10s"
role="Master"/>
<op id="op-drbd0-2" name="monitor" interval="60s" timeout="10s"
role="Slave"/>
</operations>
<meta_attributes id="drbd0:0_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
</master_slave>
<group id="mysql-group">
<meta_attributes id="ma-mysql">
<attributes/>
</meta_attributes>
<primitive class="ocf" provider="heartbeat" type="Filesystem" id="fs0">
<instance_attributes id="ia-fs0">
<attributes>
<nvpair id="ia-fs0-1" name="fstype" value="ext3"/>
<nvpair id="ia-fs0-2" name="directory" value="/mnt/drbd"/>
<nvpair id="ia-fs0-3" name="device" value="/dev/drbd0"/>
</attributes>
</instance_attributes>
<meta_attributes id="fs0_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
<primitive class="lsb" provider="heartbeat" type="httpd"
id="webserver">
<meta_attributes id="ma-webserver">
<attributes/>
</meta_attributes>
</primitive>
<primitive id="virtual-ip-httpd" class="ocf" type="IPaddr2"
provider="heartbeat">
<instance_attributes id="ia-virtual-ip-httpd">
<attributes>
<nvpair id="ia-virtual-ip-httpd-1" name="ip"
value="10.102.43.52"/>
<nvpair id="ia-virtual-ip-httpd-2" name="broadcast"
value="10.102.43.255"/>
<nvpair id="ia-virtual-ip-httpd-3" name="nic" value="bond0"/>
<nvpair id="ia-virtual-ip-httpd-4" name="cidr_netmask"
value="24"/>
<nvpair id="b5a0de6b-72c9-4d28-9772-6666dc1c73d5" name="iflabel"
value="mysql"/>
</attributes>
</instance_attributes>
<operations>
<op id="op-virtual-ip-1" name="monitor" interval="21s" timeout="5s"
start_delay="0" disabled="false" role="Started"/>
</operations>
<meta_attributes id="virtual-ip-httpd_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
<primitive id="mysql" class="ocf" type="mysql" provider="heartbeat">
<instance_attributes id="mysql_instance_attrs">
<attributes>
<nvpair id="9be7d068-044e-4edf-b871-53cb84107523" name="binary"
value="/usr/bin/mysqld_safe"/>
<nvpair id="67979188-8fe6-458e-af30-4e905c0d0105" name="datadir"
value="/mnt/drbd/mysql"/>
<nvpair id="ce64d4ba-d120-4ea2-ab94-aaafc0d61a5e" name="log"
value="/var/log/mysqld.log"/>
<nvpair id="df7b5e14-2292-4201-8911-f5c1fc131f82" name="user"
value="mysql"/>
<nvpair id="35e7d090-4f6b-4cb6-bc17-a84009a130c9" name="pid"
value="/var/run/mysqld/mysqld.pid"/>
<nvpair id="27b36f5a-a8e8-44d4-9753-194b5670cac5"
name="test_passwd" value="recorder"/>
</attributes>
</instance_attributes>
<meta_attributes id="mysql_meta_attrs">
<attributes>
<nvpair id="mysql_metaattr_target_role" name="target_role"
value="started"/>
</attributes>
</meta_attributes>
<operations>
<op id="6428333e-7547-4a16-94ee-b5ee8881d7f7" name="monitor"
interval="10" timeout="30" start_delay="10" disabled="false" role="Started"/>
</operations>
</primitive>
</group>
<group id="tomcat-group">
<primitive id="fs1" class="ocf" type="Filesystem" provider="heartbeat">
<instance_attributes id="fs1_instance_attrs">
<attributes>
<nvpair id="ee857165-d3b0-4c16-a43e-709d4d56d240" name="device"
value="/dev/drbd1"/>
<nvpair id="54014116-898f-40f6-83f8-270f455fa0de"
name="directory" value="/mnt/tomcat6"/>
<nvpair id="2466989f-eba4-4286-bd11-2043866a4270" name="fstype"
value="ext3"/>
</attributes>
</instance_attributes>
<meta_attributes id="fs1_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
<primitive class="ocf" type="IPaddr2" provider="heartbeat"
id="virtual-ip-tomcat">
<instance_attributes id="ia-virtual-ip-tomcat">
<attributes>
<nvpair id="ia-virtual-ip-tomcat-1" name="ip"
value="10.102.43.51"/>
<nvpair id="ia-virtual-ip-tomcat-2" name="nic" value="bond0"/>
<nvpair id="ia-virtual-ip-tomcat-3" name="cidr_netmask"
value="24"/>
<nvpair id="ia-virtual-ip-tomcat-4" name="broadcast"
value="10.102.43.255"/>
<nvpair id="ia-virtual-ip-tomcat-5" name="iflabel"
value="tomcat"/>
</attributes>
</instance_attributes>
<operations>
<op id="op-virtual-ip-tomcat-1" name="monitor" interval="10s"
timeout="20s" start_delay="5s" disabled="false" role="Started"/>
</operations>
<meta_attributes id="virtual-ip-tomcat_meta_attrs">
<attributes>
<nvpair id="virtual-ip-tomcat_metaattr_target_role"
name="target_role" value="started"/>
</attributes>
</meta_attributes>
</primitive>
<primitive id="tomcat6" class="ocf" type="tomcat" provider="heartbeat">
<instance_attributes id="ia-tomcat6">
<attributes>
<nvpair id="ia-tomcat6-1" name="java_home" value="/usr"/>
<nvpair id="ia-tomcat6-2" name="catalina_home"
value="/usr/share/tomcat6"/>
</attributes>
</instance_attributes>
<operations>
<op id="op-tomcat6" name="monitor" interval="10s" timeout="30s"
start_delay="10s" disabled="false" role="Started"/>
</operations>
<meta_attributes id="tomcat6_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
<meta_attributes id="tomcat-group_meta_attrs">
<attributes>
<nvpair id="tomcat-group_metaattr_target_role" name="target_role"
value="started"/>
</attributes>
</meta_attributes>
</group>
<master_slave id="ms-drbd1">
<meta_attributes id="ma-ms-drbd1">
<attributes>
<nvpair id="ma-ms-drbd1-1" name="clone_max" value="2"/>
<nvpair id="ma-ms-drbd1-2" name="clone_node_max" value="1"/>
<nvpair id="ma-ms-drbd1-3" name="master_max" value="1"/>
<nvpair id="ma-ms-drbd1-4" name="master_node_max" value="1"/>
<nvpair id="ma-ms-drbd1-5" name="notify" value="yes"/>
<nvpair id="ma-ms-drbd1-6" name="globally_unique" value="false"/>
<nvpair id="ma-ms-drbd1-7" name="target_role" value="started"/>
</attributes>
</meta_attributes>
<primitive id="drbd1" class="ocf" provider="heartbeat" type="drbd">
<instance_attributes id="ia-drbd1">
<attributes>
<nvpair id="ia-drbd1-1" name="drbd_resource" value="drbd1"/>
</attributes>
</instance_attributes>
<operations>
<op id="op-drbd1-1" name="monitor" interval="59s" timeout="10s"
role="Master"/>
<op id="op-drbd1-2" name="monitor" interval="60s" timeout="10s"
role="Slave"/>
</operations>
<meta_attributes id="drbd1:0_meta_attrs">
<attributes/>
</meta_attributes>
</primitive>
</master_slave>
</resources>
<constraints>
<rsc_order id="drbd0-before-httpd-group" from="mysql-group"
action="start" to="ms-drbd0" to_action="promote"/>
<rsc_colocation id="mysql-group-on-drbd0" to="ms-drbd0" to_role="master"
from="mysql-group" score="infinity"/>
<rsc_colocation id="tomcat-group-on-drbd1" to="ms-drbd1" to_role="master"
from="tomcat-group" score="infinity"/>
<rsc_order id="drbd1-before-tomcat-group" from="tomcat-group"
action="start" to="ms-drbd1" to_action="promote"/>
<rsc_location id="drbd1-master-on-Node2" rsc="ms-drbd1">
<rule id="prefered_drbd1-master-on-Node2" score="-100">
<expression attribute="#uname" id="exp-drbd1-master-on-Node2-1"
operation="eq" value="SAN-bal2"/>
</rule>
</rsc_location>
<rsc_location id="drbd0-master-on-Node1" rsc="ms-drbd0">
<rule id="prefered_drbd0-master-on-Node1" score="-100">
<expression attribute="#uname" id="exp-drbd0-master-on-Node1-1"
operation="eq" value="SAN-bal1"/>
</rule>
</rsc_location>
</constraints>
</configuration>
</cib>
--------------------------------------------
Jan 21 14:47:04 SAN-bal1 last message repeated 5 times
Jan 21 14:47:23 SAN-bal1 mgmtd: [5559]: info: (delete)xml:<rsc_location
id="drbd1-master-on-Node2">
Jan 21 14:47:24 SAN-bal1 mgmtd: [5559]: info: on_update_constraint:<rsc_location id="drbd1-master-on-Node2" rsc="ms-drbd1"><rule id="prefered_drbd1-master-on-Node2" score="-100"
boolean_op=""><expression attribute="#uname" id="exp-drbd1-master-on-Node2-1" operation="eq" value="SAN-bal2"/></rule></rsc_location>
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml.last (digest:
/var/lib/heartbeat/crm/cib.xml.sig.last)
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: write_cib_contents: Wrote version
0.301.1 of the CIB to disk (digest: 942c28b85c6fe6464db19a9fa562e8e7)
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:24 SAN-bal1 cib: [1858]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml.last (digest:
/var/lib/heartbeat/crm/cib.xml.sig.last)
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: send_direct_ack: ACK'ing resource
op drbd1:1_monitor_60000 from 8:17:4683aafc-818c-4601-9629-32b8976a370b:
lrm_invoke-lrmd-1232545645-7
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=124:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:25 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=126:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_monitor_60000 (call=76, rc=-2) Cancelled
Jan 21 14:47:25 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:25 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=77, rc=0) complete
Jan 21 14:47:25 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:25 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:25 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=78, rc=0) complete
Jan 21 14:47:25 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml.last (digest:
/var/lib/heartbeat/crm/cib.xml.sig.last)
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: write_cib_contents: Wrote version
0.302.3 of the CIB to disk (digest: eeceac6171eb8601486a1348241747a2)
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml (digest:
/var/lib/heartbeat/crm/cib.xml.sig)
Jan 21 14:47:25 SAN-bal1 cib: [1883]: info: retrieveCib: Reading cluster
configuration from: /var/lib/heartbeat/crm/cib.xml.last (digest:
/var/lib/heartbeat/crm/cib.xml.sig.last)
Jan 21 14:47:26 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:26 SAN-bal1 last message repeated 2 times
Jan 21 14:47:27 SAN-bal1 kernel: drbd1: peer( Primary -> Secondary )
Jan 21 14:47:27 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=127:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:27 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:27 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:27 SAN-bal1 last message repeated 2 times
Jan 21 14:47:27 SAN-bal1 crm_master: [1925]: info: Invoked:
/usr/sbin/crm_master -v 75 -l reboot
Jan 21 14:47:27 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:28 SAN-bal1 last message repeated 4 times
Jan 21 14:47:28 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:28 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=79, rc=0) complete
Jan 21 14:47:29 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=122:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:29 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:29 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=80, rc=0) complete
Jan 21 14:47:29 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:29 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: peer( Secondary -> Unknown ) conn( Connected
-> TearDown ) pdsk( UpToDate -> DUnknown )
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: meta connection shut down by peer.
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: asender terminated
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: Terminating asender thread
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: tl_clear()
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: Connection closed
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: conn( TearDown -> Unconnected )
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: receiver terminated
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: Restarting receiver thread
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: receiver (re)started
Jan 21 14:47:29 SAN-bal1 kernel: drbd1: conn( Unconnected -> WFConnection )
Jan 21 14:47:30 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_promote_0 key=71:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:30 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: promote
Jan 21 14:47:30 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=123:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:30 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:30 SAN-bal1 kernel: drbd1: role( Secondary -> Primary )
Jan 21 14:47:30 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:30 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:30 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:30 SAN-bal1 kernel: drbd1: Creating new current UUID
Jan 21 14:47:30 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:30 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:30 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:promote:stdout)
Jan 21 14:47:30 SAN-bal1 drbd[1940]: [2014]: INFO: drbd1 promote: primary
succeeded
Jan 21 14:47:30 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:30 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_promote_0 (call=81, rc=0) complete
Jan 21 14:47:31 SAN-bal1 crm_master: [2056]: info: Invoked:
/usr/sbin/crm_master -v 10 -l reboot
Jan 21 14:47:31 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=125:17:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:31 SAN-bal1 lrmd: [5555]: info: perform_op:2792: operations on
resource drbd1:1 already delayed
Jan 21 14:47:31 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:31 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:31 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=82, rc=0) complete
Jan 21 14:47:31 SAN-bal1 crm_master: [2100]: info: Invoked:
/usr/sbin/crm_master -v 10 -l reboot
Jan 21 14:47:31 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:31 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:32 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:32 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:32 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:32 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=83, rc=0) complete
Jan 21 14:47:33 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=120:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:33 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:33 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=84, rc=0) complete
Jan 21 14:47:33 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:34 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_demote_0 key=64:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:34 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: demote
Jan 21 14:47:34 SAN-bal1 kernel: drbd1: role( Primary -> Secondary )
Jan 21 14:47:34 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:34 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:34 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:demote:stdout)
Jan 21 14:47:34 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_demote_0 (call=85, rc=0) complete
Jan 21 14:47:35 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=121:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:35 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:35 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:35 SAN-bal1 crm_master: [2186]: info: Invoked:
/usr/sbin/crm_master -v 10 -l reboot
Jan 21 14:47:36 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:36 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:36 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=86, rc=0) complete
Jan 21 14:47:37 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: notify
Jan 21 14:47:37 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_notify_0 key=119:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:37 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:37 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_notify_0 (call=87, rc=0) complete
Jan 21 14:47:38 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:1_stop_0 key=65:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:38 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:1: stop
Jan 21 14:47:38 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:38 SAN-bal1 crm_master: [2241]: info: Invoked:
/usr/sbin/crm_master -D
Jan 21 14:47:39 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:39 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:stop:stdout)
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: conn( WFConnection -> Disconnecting )
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: Discarding network configuration.
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: tl_clear()
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: Connection closed
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: conn( Disconnecting -> StandAlone )
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: receiver terminated
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: Terminating receiver thread
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: disk( UpToDate -> Diskless )
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: drbd_bm_resize called with capacity == 0
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: worker terminated
Jan 21 14:47:39 SAN-bal1 kernel: drbd1: Terminating worker thread
Jan 21 14:47:39 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:1:stop:stdout)
Jan 21 14:47:39 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:1_stop_0 (call=88, rc=0) complete
Jan 21 14:47:40 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_start_0 key=61:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:40 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:0: start
Jan 21 14:47:40 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: disk( Diskless -> Attaching )
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: Starting worker thread (from cqueue/2
[234])
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: Found 6 transactions (13 active
extents) in activity log.
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: max_segment_size ( = BIO size ) = 32768
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: drbd_bm_resize called with capacity ==
22994104
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: resync bitmap: bits=2874263 words=44911
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: size = 11 GB (11497052 KB)
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: reading of bitmap took 8 jiffies
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: recounting of set bits took additional
0 jiffies
Jan 21 14:47:40 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:0:start:stdout)
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: 0 KB (0 bits) marked out-of-sync by on
disk bit-map.
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: disk( Attaching -> UpToDate )
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: Barriers not supported on meta data
device - disabling
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: conn( StandAlone -> Unconnected )
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: Starting receiver thread (from
drbd1_worker [2323])
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: receiver (re)started
Jan 21 14:47:40 SAN-bal1 kernel: drbd1: conn( Unconnected -> WFConnection )
Jan 21 14:47:40 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_start_0 (call=89, rc=0) complete
Jan 21 14:47:41 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_notify_0 key=116:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:41 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:0: notify
Jan 21 14:47:41 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:41 SAN-bal1 crm_master: [2401]: info: Invoked:
/usr/sbin/crm_master -v 10 -l reboot
Jan 21 14:47:42 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:42 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:0:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:42 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_notify_0 (call=90, rc=0) complete
Jan 21 14:47:43 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_notify_0 key=117:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:43 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:0: notify
Jan 21 14:47:43 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:43 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_notify_0 (call=91, rc=0) complete
Jan 21 14:47:44 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_promote_0 key=62:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:44 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:0: promote
Jan 21 14:47:44 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:44 SAN-bal1 kernel: drbd1: role( Secondary -> Primary )
Jan 21 14:47:44 SAN-bal1 kernel: drbd1: Writing meta data super block now.
Jan 21 14:47:44 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:0:promote:stdout)
Jan 21 14:47:44 SAN-bal1 drbd[2453]: [2527]: INFO: drbd1 promote: primary
succeeded
Jan 21 14:47:44 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_promote_0 (call=92, rc=0) complete
Jan 21 14:47:45 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_notify_0 key=118:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:45 SAN-bal1 lrmd: [5555]: info: rsc:drbd1:0: notify
Jan 21 14:47:45 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:45 SAN-bal1 crm_master: [2569]: info: Invoked:
/usr/sbin/crm_master -v 10 -l reboot
Jan 21 14:47:46 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:46 SAN-bal1 lrmd: [5555]: info: RA output: (drbd1:0:notify:stdout)
No set matching id=master-ee0f5ccd-ef1f-4dc6-8bfb-7fa6382455c7 in status
Jan 21 14:47:46 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_notify_0 (call=93, rc=0) complete
Jan 21 14:47:47 SAN-bal1 lrmd: [5555]: info: rsc:fs1: start
Jan 21 14:47:47 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=fs1_start_0 key=52:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:47 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=drbd1:0_monitor_59000 key=63:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:47 SAN-bal1 Filesystem[2572]: [2626]: INFO: Running start for
/dev/drbd1 on /mnt/tomcat6
Jan 21 14:47:47 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
drbd1:0_monitor_59000 (call=95, rc=8) complete
Jan 21 14:47:47 SAN-bal1 kernel: kjournald starting. Commit interval 5 seconds
Jan 21 14:47:47 SAN-bal1 kernel: EXT3 FS on drbd1, internal journal
Jan 21 14:47:47 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
fs1_start_0 (call=94, rc=0) complete
Jan 21 14:47:47 SAN-bal1 kernel: EXT3-fs: mounted filesystem with ordered data
mode.
Jan 21 14:47:47 SAN-bal1 haclient: on_event: from message queue: evt:cib_changed
Jan 21 14:47:48 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=virtual-ip-tomcat_start_0 key=53:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:48 SAN-bal1 lrmd: [5555]: info: rsc:virtual-ip-tomcat: start
Jan 21 14:47:48 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:48 SAN-bal1 IPaddr2[2655]: [2684]: INFO: ip -f inet addr add
10.102.43.51/24 brd 10.102.43.255 dev bond0 label bond0:tomcat
Jan 21 14:47:48 SAN-bal1 IPaddr2[2655]: [2686]: INFO: ip link set bond0 up
Jan 21 14:47:48 SAN-bal1 IPaddr2[2655]: [2688]: INFO: /usr/lib64/heartbeat/send_arp -i 200 -r 5 -p /var/run/heartbeat/rsctmp/send_arp/send_arp-10.102.43.51 bond0 10.102.43.51 auto
not_used not_used
Jan 21 14:47:48 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
virtual-ip-tomcat_start_0 (call=96, rc=0) complete
Jan 21 14:47:48 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:49 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=virtual-ip-tomcat_monitor_10000
key=54:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:49 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=tomcat6_start_0 key=55:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:49 SAN-bal1 lrmd: [5555]: info: rsc:tomcat6: start
Jan 21 14:47:49 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:52 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
tomcat6_start_0 (call=98, rc=0) complete
Jan 21 14:47:53 SAN-bal1 crmd: [5558]: info: do_lrm_rsc_op: Performing
op=tomcat6_monitor_10000 key=56:18:4683aafc-818c-4601-9629-32b8976a370b)
Jan 21 14:47:53 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:47:54 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
virtual-ip-tomcat_monitor_10000 (call=97, rc=0) complete
Jan 21 14:47:55 SAN-bal1 haclient: on_event:evt:cib_changed
Jan 21 14:48:03 SAN-bal1 crmd: [5558]: info: process_lrm_event: LRM operation
tomcat6_monitor_10000 (call=99, rc=0) complete
Jan 21 14:48:04 SAN-bal1 haclient: on_event:evt:cib_changed
Le informazioni trasmesse sono destinate esclusivamente alla persona o alla
societa' in indirizzo e sono da intendersi confidenziali e riservate. Ogni
trasmissione, inoltro, diffusione o altro uso di queste informazioni a persone
o societa' differenti dal destinatario e' proibita. Se ricevete questa
comunicazione per errore, contattate il mittente e cancellate le informazioni
da ogni computer.
The information transmitted is intended only for the person or entity to which
it is addressed and may contain confidential and/or privileged material. Any
review, retransmission, dissemination or other use of, or taking of any action
in reliance upon, this information by persons or entities other than the
intended recipient is prohibited. If you received this in error, please contact
the sender and delete the material from any computer
_______________________________________________
Linux-HA mailing list
[email protected]
http://lists.linux-ha.org/mailman/listinfo/linux-ha
See also: http://linux-ha.org/ReportingProblems
