Mohit I've tried same test and reproduce the 'Peer Rejected' status. please find config files and log files in attached taz.
[root@vhead-010 ~]# date
Tue Aug 16 11:55:15 JST 2011
[root@vhead-010 ~]# cat a.sh
#!/bin/bash
for i in foo-{1..3}-private
do
ssh ${i} service glusterd stop
ssh ${i} 'find /etc/glusterd -type f|xargs rm -f'
ssh ${i} rm -rf /etc/glusterd/vols/*
ssh ${i} service iptables stop
ssh ${i} cp /dev/null /var/log/glusterfs/nfs.log
ssh ${i} cp /dev/null /var/log/glusterfs/bricks/mnt-brick.log
ssh ${i} cp /dev/null /var/log/glusterfs/.cmd_log_history
ssh ${i} cp /dev/null /var/log/glusterfs/etc-glusterfs-glusterd.vol.log
ssh ${i} service glusterd start
ssh ${i} find /etc/glusterd
ssh ${i} service glusterd status
done
[root@vhead-010 ~]# bash a.sh
Stopping glusterd:[ OK ]
Flushing firewall rules: [ OK ]
Setting chains to policy ACCEPT: filter [ OK ]
Unloading iptables modules: [ OK ]
Starting glusterd:[ OK ]
/etc/glusterd
/etc/glusterd/glusterd.info
/etc/glusterd/nfs
/etc/glusterd/nfs/run
/etc/glusterd/peers
/etc/glusterd/vols
glusterd (pid 15617) is running...
Stopping glusterd:[ OK ]
Flushing firewall rules: [ OK ]
Setting chains to policy ACCEPT: filter [ OK ]
Unloading iptables modules: [ OK ]
Starting glusterd:[ OK ]
/etc/glusterd
/etc/glusterd/glusterd.info
/etc/glusterd/nfs
/etc/glusterd/nfs/run
/etc/glusterd/peers
/etc/glusterd/vols
glusterd (pid 15147) is running...
Stopping glusterd:[ OK ]
Flushing firewall rules: [ OK ]
Setting chains to policy ACCEPT: filter [ OK ]
Unloading iptables modules: [ OK ]
Starting glusterd:[ OK ]
/etc/glusterd
/etc/glusterd/glusterd.info
/etc/glusterd/nfs
/etc/glusterd/nfs/run
/etc/glusterd/peers
/etc/glusterd/vols
glusterd (pid 15177) is running...
[root@vhead-010 ~]# ssh foo-1-private
Last login: Tue Aug 16 09:51:49 2011 from dlp.local.valinux.co.jp
[root@localhost ~]# gluster peer probe foo-2-private
Probe successful
[root@localhost ~]# gluster peer status
Number of Peers: 1
Hostname: foo-2-private
Uuid: 20b73d9a-ede0-454f-9fbb-b0eee9ce26a3
State: Peer in Cluster (Connected)
[root@localhost ~]# gluster volume create foo foo-1-private:/mnt/brick
Creation of volume foo has been successful. Please start the volume to access
data.
[root@localhost ~]# gluster volume start foo
Starting volume foo has been successful
[root@localhost ~]# gluster volume add-brick foo foo-2-private:/mnt/brick
Add Brick successful
[root@localhost ~]# gluster peer probe foo-3-private
Probe successful
[root@localhost ~]# gluster peer status
Number of Peers: 2
Hostname: foo-2-private
Uuid: 20b73d9a-ede0-454f-9fbb-b0eee9ce26a3
State: Peer in Cluster (Connected)
Hostname: foo-3-private
Uuid: 7587ae34-9209-484a-9576-3939e061720c
State: Peer Rejected (Connected)
[root@localhost ~]# exit
logout
Connection to foo-1-private closed.
[root@vhead-010 ~]# find foo_log_and_conf
foo_log_and_conf
foo_log_and_conf/foo-2-private
foo_log_and_conf/foo-2-private/glusterd
foo_log_and_conf/foo-2-private/glusterd/glusterd.info
foo_log_and_conf/foo-2-private/glusterd/nfs
foo_log_and_conf/foo-2-private/glusterd/nfs/nfs-server.vol
foo_log_and_conf/foo-2-private/glusterd/nfs/run
foo_log_and_conf/foo-2-private/glusterd/nfs/run/nfs.pid
foo_log_and_conf/foo-2-private/glusterd/peers
foo_log_and_conf/foo-2-private/glusterd/peers/461f6e21-90c4-4b6c-bda8-7b99bacb2722
foo_log_and_conf/foo-2-private/glusterd/vols
foo_log_and_conf/foo-2-private/glusterd/vols/foo
foo_log_and_conf/foo-2-private/glusterd/vols/foo/info
foo_log_and_conf/foo-2-private/glusterd/vols/foo/bricks
foo_log_and_conf/foo-2-private/glusterd/vols/foo/bricks/foo-2-private:-mnt-brick
foo_log_and_conf/foo-2-private/glusterd/vols/foo/bricks/foo-1-private:-mnt-brick
foo_log_and_conf/foo-2-private/glusterd/vols/foo/foo.foo-2-private.mnt-brick.vol
foo_log_and_conf/foo-2-private/glusterd/vols/foo/cksum
foo_log_and_conf/foo-2-private/glusterd/vols/foo/run
foo_log_and_conf/foo-2-private/glusterd/vols/foo/run/foo-2-private-mnt-brick.pid
foo_log_and_conf/foo-2-private/glusterd/vols/foo/foo-fuse.vol
foo_log_and_conf/foo-2-private/glusterd/vols/foo/foo.foo-1-private.mnt-brick.vol
foo_log_and_conf/foo-2-private/glusterfs
foo_log_and_conf/foo-2-private/glusterfs/nfs.log
foo_log_and_conf/foo-2-private/glusterfs/bricks
foo_log_and_conf/foo-2-private/glusterfs/bricks/mnt-brick.log
foo_log_and_conf/foo-2-private/glusterfs/.cmd_log_history
foo_log_and_conf/foo-2-private/glusterfs/etc-glusterfs-glusterd.vol.log
foo_log_and_conf/foo-1-private
foo_log_and_conf/foo-1-private/glusterd
foo_log_and_conf/foo-1-private/glusterd/glusterd.info
foo_log_and_conf/foo-1-private/glusterd/nfs
foo_log_and_conf/foo-1-private/glusterd/nfs/nfs-server.vol
foo_log_and_conf/foo-1-private/glusterd/nfs/run
foo_log_and_conf/foo-1-private/glusterd/nfs/run/nfs.pid
foo_log_and_conf/foo-1-private/glusterd/peers
foo_log_and_conf/foo-1-private/glusterd/peers/20b73d9a-ede0-454f-9fbb-b0eee9ce26a3
foo_log_and_conf/foo-1-private/glusterd/peers/7587ae34-9209-484a-9576-3939e061720c
foo_log_and_conf/foo-1-private/glusterd/vols
foo_log_and_conf/foo-1-private/glusterd/vols/foo
foo_log_and_conf/foo-1-private/glusterd/vols/foo/info
foo_log_and_conf/foo-1-private/glusterd/vols/foo/bricks
foo_log_and_conf/foo-1-private/glusterd/vols/foo/bricks/foo-2-private:-mnt-brick
foo_log_and_conf/foo-1-private/glusterd/vols/foo/bricks/foo-1-private:-mnt-brick
foo_log_and_conf/foo-1-private/glusterd/vols/foo/foo.foo-2-private.mnt-brick.vol
foo_log_and_conf/foo-1-private/glusterd/vols/foo/cksum
foo_log_and_conf/foo-1-private/glusterd/vols/foo/run
foo_log_and_conf/foo-1-private/glusterd/vols/foo/run/foo-1-private-mnt-brick.pid
foo_log_and_conf/foo-1-private/glusterd/vols/foo/foo-fuse.vol
foo_log_and_conf/foo-1-private/glusterd/vols/foo/foo.foo-1-private.mnt-brick.vol
foo_log_and_conf/foo-1-private/glusterfs
foo_log_and_conf/foo-1-private/glusterfs/nfs.log
foo_log_and_conf/foo-1-private/glusterfs/bricks
foo_log_and_conf/foo-1-private/glusterfs/bricks/mnt-brick.log
foo_log_and_conf/foo-1-private/glusterfs/.cmd_log_history
foo_log_and_conf/foo-1-private/glusterfs/etc-glusterfs-glusterd.vol.log
foo_log_and_conf/foo-3-private
foo_log_and_conf/foo-3-private/glusterd
foo_log_and_conf/foo-3-private/glusterd/glusterd.info
foo_log_and_conf/foo-3-private/glusterd/nfs
foo_log_and_conf/foo-3-private/glusterd/nfs/run
foo_log_and_conf/foo-3-private/glusterd/peers
foo_log_and_conf/foo-3-private/glusterd/peers/461f6e21-90c4-4b6c-bda8-7b99bacb2722
foo_log_and_conf/foo-3-private/glusterd/vols
foo_log_and_conf/foo-3-private/glusterd/vols/foo
foo_log_and_conf/foo-3-private/glusterd/vols/foo/info
foo_log_and_conf/foo-3-private/glusterd/vols/foo/bricks
foo_log_and_conf/foo-3-private/glusterd/vols/foo/bricks/foo-2-private:-mnt-brick
foo_log_and_conf/foo-3-private/glusterd/vols/foo/bricks/foo-1-private:-mnt-brick
foo_log_and_conf/foo-3-private/glusterd/vols/foo/foo.foo-2-private.mnt-brick.vol
foo_log_and_conf/foo-3-private/glusterd/vols/foo/cksum
foo_log_and_conf/foo-3-private/glusterd/vols/foo/foo-fuse.vol
foo_log_and_conf/foo-3-private/glusterd/vols/foo/foo.foo-1-private.mnt-brick.vol
foo_log_and_conf/foo-3-private/glusterfs
foo_log_and_conf/foo-3-private/glusterfs/nfs.log
foo_log_and_conf/foo-3-private/glusterfs/bricks
foo_log_and_conf/foo-3-private/glusterfs/bricks/mnt-brick.log
foo_log_and_conf/foo-3-private/glusterfs/.cmd_log_history
foo_log_and_conf/foo-3-private/glusterfs/etc-glusterfs-glusterd.vol.log
[root@vhead-010 ~]# exit
Best,
(2011/08/16 9:35), Mohit Anchlia wrote:
I should have also asked you to stop and delete volume before getting rid of gluster config files. Can you get rid of directories also inside vols and try to restart? It's trying to look for volume files that we just removed. Also, just disable iptables for now explicitly. On Mon, Aug 15, 2011 at 5:22 PM, Tomoaki Sato<[email protected]> wrote:1) run peer detach for all the serversdone.2) from server 1 ->3 and 3->1 make sure ports are open and iptables are turned off.done. by the way, the same test on 3.1.5-1 works fine with same environment.3) remove config files under /etc/glusterdplease review following logs.4) run your tests again.I don't know why but glusterd service failed to start on all 3 hosts. [root@vhead-010 ~]# date Tue Aug 16 09:12:53 JST 2011 [root@vhead-010 ~]# cat a.sh #!/bin/bash for i in foo-{1..3}-private do ssh ${i} service glusterd stop ssh ${i} 'find /etc/glusterd -type f|xargs rm -f' ssh ${i} service iptables restart ssh ${i} iptables -vL ssh ${i} service glusterd start ssh ${i} find /etc/glusterd ssh ${i} service glusterd status done [root@vhead-010 ~]# bash a.sh Stopping glusterd:[ OK ] Flushing firewall rules: [ OK ] Setting chains to policy ACCEPT: filter [ OK ] Unloading iptables modules: [ OK ] Chain INPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain FORWARD (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols /etc/glusterd/vols/foo /etc/glusterd/vols/foo/bricks /etc/glusterd/vols/foo/run glusterd is stopped Stopping glusterd:[ OK ] Flushing firewall rules: [ OK ] Setting chains to policy ACCEPT: filter [ OK ] Unloading iptables modules: [ OK ] Chain INPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain FORWARD (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols /etc/glusterd/vols/foo /etc/glusterd/vols/foo/bricks /etc/glusterd/vols/foo/run glusterd is stopped Stopping glusterd:[ OK ] Flushing firewall rules: [ OK ] Setting chains to policy ACCEPT: filter [ OK ] Unloading iptables modules: [ OK ] Chain INPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain FORWARD (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols /etc/glusterd/vols/foo /etc/glusterd/vols/foo/bricks /etc/glusterd/vols/foo/run glusterd is stopped [root@vhead-010 ~]# date Tue Aug 16 09:13:20 JST 2011 [root@vhead-010 ~]# ssh foo-1-private Last login: Tue Aug 16 09:06:57 2011 from dlp.local.valinux.co.jp [root@localhost ~]# tail -20 /var/log/glusterfs/etc-glusterfs-glusterd.vol.log ... [2011-08-16 09:13:01.85858] I [glusterd.c:304:init] 0-management: Using /etc/glusterd as working directory [2011-08-16 09:13:01.87294] E [rpc-transport.c:799:rpc_transport_load] 0-rpc-transport: /opt/glusterfs/3.1.6/lib64/glusterfs/3.1.6/rpc-transport/rdma.so: cannot open shared object file: No such file or directory [2011-08-16 09:13:01.87340] E [rpc-transport.c:803:rpc_transport_load] 0-rpc-transport: volume 'rdma.management': transport-type 'rdma' is not valid or not found on this machine [2011-08-16 09:13:01.87402] E [glusterd-store.c:654:glusterd_store_handle_retrieve] 0-glusterd: Unable to retrieve store handle for /etc/glusterd/glusterd.info, error: No such file or directory [2011-08-16 09:13:01.87422] E [glusterd-store.c:761:glusterd_retrieve_uuid] 0-: Unable to get store handle! [2011-08-16 09:13:01.87514] I [glusterd.c:95:glusterd_uuid_init] 0-glusterd: generated UUID: c0cef9f9-a79e-4189-8955-d83927db9cee [2011-08-16 09:13:01.87681] E [glusterd-store.c:654:glusterd_store_handle_retrieve] 0-glusterd: Unable to retrieve store handle for /etc/glusterd/vols/foo/info, error: No such file or directory [2011-08-16 09:13:01.87704] E [glusterd-store.c:1328:glusterd_store_retrieve_volumes] 0-: Unable to restore volume: foo[2011-08-16 09:13:01.87732] E [xlator.c:843:xlator_init] 0-management: Initialization of volume 'management' failed, review your volfile again [2011-08-16 09:13:01.87751] E [graph.c:331:glusterfs_graph_init] 0-management: initializing translator failed [2011-08-16 09:13:01.87818] I [glusterfsd.c:712:cleanup_and_exit] 0-glusterfsd: shutting down [root@localhost ~]# exit Best, (2011/08/16 8:52), Mohit Anchlia wrote:Logs are generally in /var/log/gluster Since you are playing with it. I would suggest this: 1) run peer detach for all the servers 2) from server 1 ->3 and 3->1 make sure ports are open and iptables are turned off. 3) remove config files under /etc/glusterd 4) run your tests again. On Mon, Aug 15, 2011 at 4:28 PM, Tomoaki Sato<[email protected]> wrote:Thanks, Mohit (2011/08/16 8:05), Mohit Anchlia wrote:What's in your logs?I can obtain logs needed. could you tell me the instruction to take the logs?Did you have foo-3-private before in your gluster cluster ever or adding this host for the first time?It was first time. All foo-X-private has no entries in /etc/glusterd/peers/ and /etc/glusterd/vols/.Try gluster peer detach and then remove any left over configuration in /etc/glusterd config directory. After that try again and see if that works.[root@vhead-010 ~]# date Tue Aug 16 08:17:49 JST 2011 [root@vhead-010 ~]# cat a.sh #!/bin/bash for i in foo-{1..3}-private do ssh ${i} service glusterd stop ssh ${i} rm -rf /etc/glusterd/peers/* ssh ${i} rm -rf /etc/glusterd/vols/* ssh ${i} service glusterd start ssh ${i} find /etc/glusterd done [root@vhead-010 ~]# bash a.sh Stopping glusterd:[ OK ] Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/nfs-server.vol /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols Stopping glusterd:[ OK ] Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/nfs-server.vol /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols Stopping glusterd:[ OK ] Starting glusterd:[ OK ] /etc/glusterd /etc/glusterd/glusterd.info /etc/glusterd/nfs /etc/glusterd/nfs/nfs-server.vol /etc/glusterd/nfs/run /etc/glusterd/peers /etc/glusterd/vols [root@vhead-010 ~]# ssh foo-1-private [root@localhost ~]# gluster peer probe foo-2-private Probe successful [root@localhost ~]# gluster peer status Number of Peers: 1 Hostname: foo-2-private Uuid: c2b314ac-6ed1-455a-84d4-ec22041ee2b2 State: Peer in Cluster (Connected) [root@localhost ~]# gluster volume create foo foo-1-private:/mnt/brick Creation of volume foo has been successful. Please start the volume to access da ta. [root@localhost ~]# gluster volume start foo Starting volume foo has been successful [root@localhost ~]# gluster volume add-brick foo foo-2-private:/mnt/brick Add Brick successful [root@localhost ~]# gluster peer probe foo-3-private Probe successful [root@localhost ~]# gluster peer status Number of Peers: 2 Hostname: foo-2-private Uuid: c2b314ac-6ed1-455a-84d4-ec22041ee2b2 State: Peer in Cluster (Connected) Hostname: foo-3-private Uuid: 7fb98dac-fef7-4b33-837c-6483a767ec3e State: Peer Rejected (Connected) [root@localhost ~]# cat /var/log/glusterfs/.cmd_log_history ... [2011-08-16 08:20:28.862619] peer probe : on host foo-2-private:24007 [2011-08-16 08:20:28.912419] peer probe : on host foo-2-private:24007 FAILED [2011-08-16 08:20:58.382350] Volume create : on volname: foo attempted [2011-08-16 08:20:58.382461] Volume create : on volname: foo type:DEFAULT count: 1 bricks: foo-1-private:/mnt/brick [2011-08-16 08:20:58.384674] Volume create : on volname: foo SUCCESS [2011-08-16 08:21:04.831772] volume start : on volname: foo SUCCESS [2011-08-16 08:21:22.682292] Volume add-brick : on volname: foo attempted [2011-08-16 08:21:22.682385] Volume add-brick : volname: foo type DEFAULT count: 1 bricks: foo-2-private:/mnt/brick [2011-08-16 08:21:22.682499] Volume add-brick : on volname: foo SUCCESS [2011-08-16 08:21:39.124574] peer probe : on host foo-3-private:24007 [2011-08-16 08:21:39.135609] peer probe : on host foo-3-private:24007 FAILED TomoOn Mon, Aug 15, 2011 at 3:37 PM, Tomoaki Sato<[email protected]> wrote:Hi, following instructions work fine with 3.1.5-1 but with 3.1.6-1. 1. make a new file system without peers. [OK] foo-1-private# gluster volume create foo foo-1-private:/mnt/brick foo-1-private# gluster volume start foo foo-1-private# gluster peer status No peers present foo-1-private# 2. add a peer to the file system. [NG] foo-1-private# gluster peer probe foo-2-private Probe successful foo-1-private# gluster peer status Number of Peers: 1 Hostname: foo-2-private Uuid: c2b314ac-6ed1-455a-84d4-ec22041ee2b2 State: Peer Rejected (Connected) foo-1-private# gluster volume add-brick foo foo-2-private:/mnt/brick Host foo-2-private not connected foo-1-private# following instructions work fine even with 3.1.6-1. 1. make a new file system with single peer. [OK] foo-1-private# gluster peer status No peer presents foo-1-private# gluster peer probe foo-2-private Probe successful foo-1-private# gluster peer status Number of Peers: 1 Hostname: foo-2-private Uuid: c2b314ac-6ed1-455a-84d4-ec22041ee2b2 State: Peer in Cluster (Connected) foo-1-private# gluster volume create foo foo-1-private:/mnt/brick Creation of volume foo has been successful. Please start the volume to access data. foo-1-private# gluster volume start foo Starting volume foo has been successful foo-1-private# gluster volume add-brick foo foo-2-private:/mnt/brick Add Brick successful foo-1-private# But ... 2. add a peer to the file system. [NG] foo-1-private# gluster peer probe foo-3-private Probe successful foo-1-private# gluster peer status Number of Peers: 2 Hostname: foo-2-private Uuid: c2b314ac-6ed1-455a-84d4-ec22041ee2b2 State: Peer in Cluster (Connected) Hostname: foo-3-private Uuid: 7fb98dac-fef704b33-837c-6483a767ec3e State: Peer Rejected (Connected) foo-1-private# gluster volume add-brick foo foo-3-private:/mnt/brick Host foo-3-private not connected foo-1-private# How should I add extra peers to existing file systems ? Best, _______________________________________________ Gluster-users mailing list [email protected] http://gluster.org/cgi-bin/mailman/listinfo/gluster-users
foo_log_and_conf.taz
Description: Binary data
_______________________________________________ Gluster-users mailing list [email protected] http://gluster.org/cgi-bin/mailman/listinfo/gluster-users
