I´m trying to build a 2 node cluster with an extra qdevice to have 3
votes with proxmox and drbd.
node 1 1GB NIC 192.168.1.245 2.5GB NIC 192.168.3.1
node 1 1GB NIC 192.168.1.246 2.5GB NIC 192.168.3.2
After installing proxmox 6.4 i install drbd9/linstor.
#apt install linstor-controller linstor-satellite linstor-client
#systemctl start linstor-satellite
#systemctl enable linstor-satellite
#systemctl start linstor-controller
#systemctl enable linstor-satellite
#linstor node create proxmoxn1 192.168.3.1 --node-type Combined
#linstor node create proxmoxn2 192.168.3.2 --node-type Combined
/etc/linstor/linstor-client.conf
[global]
controllers=proxmoxn1,proxmoxn2
#create a partition with fdisk /dev/nvme0n1
#vgcreate vg_ssd /dev/nvme0n1p4
On First node
#linstor storage-pool create lvm proxmoxn1 pool_ssd vg_ssd
#linstor storage-pool create lvm proxmoxn2 pool_ssd vg_ssd
#linstor resource-group create adcgrp --storage-pool pool_ssd
--place-count 2
#linstor vg create adcgrp
On both nodes
#apt install linstor-proxmox
/etc/pve/storage.cfg
drbd: drbdstorage
content images, rootdir
controller 192.168.3.1,192.168.3.2
resourcegroup adcgrp
#systemctl restart pvedaemon
Making linstor HA
#linstor resource-definition create linstor_db
#linstor resource-definition set-property linstor_db
DrbdOptions/Resource/on-no-quorum io-error
#linstor volume-definition create linstor_db 200M
#linstor resource create linstor_db -s pool_ssd --auto-place 2
On both nodes
#systemctl disable --now linstor-controller
#cat << EOF > /etc/systemd/system/var-lib-linstor.mount
[Unit]
Description=Filesystem for the LINSTOR controller
[Mount]
# you can use the minor like /dev/drbdX or the udev symlink
What=/dev/drbd/by-res/linstor_db/0
Where=/var/lib/linstor
EOF
#mv /var/lib/linstor{,.orig}
#mkfs.ext4 /dev/drbd/by-res/linstor_db/0
#systemctl start var-lib-linstor.mount
#cp -r /var/lib/linstor.orig/* /var/lib/linstor
#systemctl start linstor-controller
#scp /etc/systemd/system/var-lib-linstor.mount
root@192.168.1.246:/etc/systemd/system/var-lib-linstor.mount
#systemctl start linstor-controller
#apt install drbd-reactor
#mkdir /etc/drbd-reactor.d
/etc/drbd-reactor.d/linstor.toml
[promoter]]
promoter.resources.linstor_db]
start = ["var-lib-linstor.mount", "linstor-controller.service"]
#systemctl restart drbd-reactor
#systemctl enable drbd-reactor
#systemctl edit linstor-satellite
[Service]
Environment=LS_KEEP_RES=linstor_db
[Unit]
After=drbd-reactor.service
#systemctl restart linstor-satellite
I can create VM's and all seems to be ok.
After rebooting both nodes linstor/drbdadm shows this behaviour
now VM is very slow (10 times slower the in proxmox LVMTHIN)
dmesg show split-brain only for linstor_db
[ 17.632010] drbd linstor_db/0 drbd1001 proxmoxn2: helper command:
/sbin/drbdadm initial-split-brain
[ 17.632621] drbd linstor_db/0 drbd1001 proxmoxn2: helper command:
/sbin/drbdadm initial-split-brain exit code 0
[ 17.632627] drbd linstor_db/0 drbd1001: Split-Brain detected but
unresolved, dropping connection!
[ 17.632646] drbd linstor_db/0 drbd1001 proxmoxn2: helper command:
/sbin/drbdadm split-brain
[ 17.633208] drbd linstor_db/0 drbd1001 proxmoxn2: helper command:
/sbin/drbdadm split-brain exit code 0
Even manually fix split-brain doesn't work
Output of the nodes
First node
root@proxmoxn1:~# linstor r l
╭────────────────────────────────────────────────────────────────────────────────────────────────────╮
┊ ResourceName ┊ Node ┊ Port ┊ Usage ┊ Conns ┊
State ┊ CreatedOn ┊
╞════════════════════════════════════════════════════════════════════════════════════════════════════╡
┊ linstor_db ┊ proxmoxn1 ┊ 7001 ┊ InUse ┊ StandAlone(proxmoxn2) ┊
UpToDate ┊ 2021-06-01 21:34:35 ┊
┊ linstor_db ┊ proxmoxn2 ┊ 7001 ┊ InUse ┊ Connecting(proxmoxn1) ┊
UpToDate ┊ 2021-06-01 21:34:35 ┊
┊ vm-100-disk-1 ┊ proxmoxn1 ┊ 7000 ┊ Unused ┊ Ok ┊
UpToDate ┊ 2021-05-29 12:30:08 ┊
┊ vm-100-disk-1 ┊ proxmoxn2 ┊ 7000 ┊ Unused ┊ Ok ┊
UpToDate ┊ 2021-05-29 12:30:07 ┊
┊ vm-108-disk-1 ┊ proxmoxn1 ┊ 7002 ┊ InUse ┊ StandAlone(proxmoxn2) ┊
UpToDate ┊ 2021-06-06 21:01:10 ┊
┊ vm-108-disk-1 ┊ proxmoxn2 ┊ 7002 ┊ Unused ┊ Connecting(proxmoxn1) ┊
UpToDate ┊ 2021-06-06 21:01:10 ┊
╰────────────────────────────────────────────────────────────────────────────────────────────────────╯
root@proxmoxn1:~# drbdadm status
linstor_db role:Primary
disk:UpToDate
proxmoxn2 connection:StandAlone
vm-100-disk-1 role:Secondary
disk:UpToDate
proxmoxn2 role:Secondary
peer-disk:UpToDate
vm-108-disk-1 role:Primary
disk:UpToDate
proxmoxn2 connection:StandAlone
root@proxmoxn1:~#
Second node
root@proxmoxn2:~# drbdadm status
linstor_db role:Primary
disk:UpToDate
proxmoxn1 connection:Connecting
vm-100-disk-1 role:Secondary
disk:UpToDate
proxmoxn1 role:Secondary
peer-disk:UpToDate
vm-108-disk-1 role:Secondary
disk:UpToDate
proxmoxn1 connection:Connecting
I´ve read the docs again an again but no luck
Can anybody help ?
Martin
_______________________________________________
Star us on GITHUB: https://github.com/LINBIT
drbd-user mailing list
drbd-user@lists.linbit.com
https://lists.linbit.com/mailman/listinfo/drbd-user
