On Sat, Jun 2, 2018 at 12:37 AM, aginwala <amgin...@gmail.com> wrote:
> using pacemaker so that controllers can be placed in different fault > domains. > More background about the discussions can be found on: > https://mail.openvswitch.org/pipermail/ovs-discuss/2018-May/046770.html > > Signed-off-by: aginwala <aginw...@ebay.com> > --- > Documentation/topics/integration.rst | 34 +++++++++++++--- > ovn/utilities/ovndb-servers.ocf | 76 +++++++++++++++++++++++++++--- > ------ > 2 files changed, 86 insertions(+), 24 deletions(-) > > diff --git a/Documentation/topics/integration.rst b/Documentation/topics/ > integration.rst > index 0447faf..5d2d3e4 100644 > --- a/Documentation/topics/integration.rst > +++ b/Documentation/topics/integration.rst > @@ -243,12 +243,14 @@ node at which the active server is run, it is not > efficient to instruct all the > ovn-controllers and the ovn-northd to listen to the latest active server's > ip-address. > > -This problem can be solved by using a native ocf resource agent > -``ocf:heartbeat:IPaddr2``. The IPAddr2 resource agent is just a resource > with > -an ip-address. When we colocate this resource with the active server, > pacemaker > -will enable the active server to be connected with a single ip-address > all the > -time. This is the ip-address that needs to be given as the parameter while > -creating the `ovndb_servers` resource. > +This problem can be solved by two ways: > + > +1. By using a native ocf resource agent ``ocf:heartbeat:IPaddr2``. > +The IPAddr2 resource agent is just a resource with an ip-address. When we > +colocate this resource with the active server, pacemaker will enable the > +active server to be connected with a single ip-address all the time. This > is > +the ip-address that needs to be given as the parameter while creating the > +`ovndb_servers` resource. > > Use the following command to create the IPAddr2 resource and colocate it > with the active server:: > @@ -258,3 +260,23 @@ with the active server:: > $ pcs constraint order promote ovndb_servers-master then VirtualIP > $ pcs constraint colocation add VirtualIP with master > ovndb_servers-master \ > score=INFINITY > + > + > +2. Using load balancer vip ip as a master_ip. > +In order to use this feature, one needs to use listen_on_master_ip_only > to no. > +Current code for load balancer have been tested to work with tcp protocol > +and needs to be tested/enchanced for ssl. Using load balancer, standby > nodes > +will not listen on nb and sb db ports so that load balancer will always > +communicate to the active node and all the traffic will be sent to active > node only. > +Standby will continue to sync using LB VIP IP in this case. > + > +Use the following command to create pcs resource using LB VIP IP:: > + > + $ pcs resource create ovndb_servers ocf:ovn:ovndb-servers \ > + master_ip="<load_balance_vip_ip>" \ > + listen_on_master_ip_only="no" \ > + ovn_ctl=<path of the ovn-ctl script> \ > + op monitor interval="10s" \ > + op monitor role=Master interval="15s" > + $ pcs resource master ovndb_servers-master ovndb_servers \ > + meta notify="true" > diff --git a/ovn/utilities/ovndb-servers.ocf > b/ovn/utilities/ovndb-servers.ocf > index 23dc700..c60ad4f 100755 > --- a/ovn/utilities/ovndb-servers.ocf > +++ b/ovn/utilities/ovndb-servers.ocf > @@ -9,6 +9,7 @@ > : ${SB_MASTER_PROTO_DEFAULT="tcp"} > : ${MANAGE_NORTHD_DEFAULT="no"} > : ${INACTIVE_PROBE_DEFAULT="5000"} > +: ${LISTEN_ON_MASTER_IP_ONLY_DEFAULT="yes"} > > CRM_MASTER="${HA_SBIN_DIR}/crm_master -l reboot" > CRM_ATTR_REPL_INFO="${HA_SBIN_DIR}/crm_attribute --type crm_config > --name OVN_REPL_INFO -s ovn_ovsdb_master_server" > @@ -21,6 +22,10 @@ SB_MASTER_PROTO=${OCF_RESKEY_sb_master_protocol:-${SB_ > MASTER_PROTO_DEFAULT}} > MANAGE_NORTHD=${OCF_RESKEY_manage_northd:-${MANAGE_NORTHD_DEFAULT}} > INACTIVE_PROBE=${OCF_RESKEY_inactive_probe_interval:-${ > INACTIVE_PROBE_DEFAULT}} > > +# In order for pacemaker to work with LB, we can set > LISTEN_ON_MASTER_IP_ONLY > +# to false and pass LB vip IP while creating pcs resource. > +LISTEN_ON_MASTER_IP_ONLY=${OCF_RESKEY_listen_on_master_ > ip_only:-${LISTEN_ON_MASTER_IP_ONLY_DEFAULT}} > + > # Invalid IP address is an address that can never exist in the network, as > # mentioned in rfc-5737. The ovsdb servers connects to this IP address > till > # a master is promoted and the IPAddr2 resource is started. > @@ -117,6 +122,16 @@ ovsdb_server_metadata() { > <content type="string" /> > </parameter> > > + <parameter name="listen_on_master_ip_only" unique="1"> > + <longdesc lang="en"> > + If set to yes, the OVNDBs will listen on master IP. Otherwise, it will > + listen on 0.0.0.0. Set to yes when using pacemaker managed vip resource > + as MASTER_IP; set to no when using external LB VIP. > + </longdesc> > + <shortdesc lang="en">Listen on master IP or 0.0.0.0</shortdesc> > + <content type="string" /> > + </parameter> > + > </parameters> > > <actions> > @@ -157,22 +172,25 @@ ovsdb_server_notify() { > ${OVN_CTL} --ovn-manage-ovsdb=no start_northd > fi > > - conn=`ovn-nbctl get NB_global . connections` > - if [ "$conn" == "[]" ] > - then > - ovn-nbctl -- --id=@conn_uuid create Connection \ > + # Not needed while listening on 0.0.0.0 as we do not want to allow > + # local binds. However, it is needed if vip ip is binded to nodes. > + if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xyes ]; then > + conn=`ovn-nbctl get NB_global . connections` > + if [ "$conn" == "[]" ] > + then > + ovn-nbctl -- --id=@conn_uuid create Connection \ > target="p${NB_MASTER_PROTO}\:${NB_MASTER_PORT}\:${MASTER_IP}" \ > inactivity_probe=$INACTIVE_PROBE -- set NB_Global . > connections=@conn_uuid > - fi > + fi > > - conn=`ovn-sbctl get SB_global . connections` > - if [ "$conn" == "[]" ] > - then > - ovn-sbctl -- --id=@conn_uuid create Connection \ > + conn=`ovn-sbctl get SB_global . connections` > + if [ "$conn" == "[]" ] > + then > + ovn-sbctl -- --id=@conn_uuid create Connection \ > target="p${SB_MASTER_PROTO}\:${SB_MASTER_PORT}\:${MASTER_IP}" \ > inactivity_probe=$INACTIVE_PROBE -- set SB_Global . > connections=@conn_uuid > + fi > fi > - > else > if [ "$MANAGE_NORTHD" = "yes" ]; then > # Stop ovn-northd service. Set --ovn-manage-ovsdb=no so that > @@ -295,15 +313,13 @@ ovsdb_server_start() { > > set ${OVN_CTL} > > - set $@ --db-nb-addr=${MASTER_IP} --db-nb-port=${NB_MASTER_PORT} > - set $@ --db-sb-addr=${MASTER_IP} --db-sb-port=${SB_MASTER_PORT} > - > - if [ "x${NB_MASTER_PROTO}" = xtcp ]; then > - set $@ --db-nb-create-insecure-remote=yes > - fi > + if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xno ]; then > + set $@ --db-nb-port=${NB_MASTER_PORT} > + set $@ --db-sb-port=${SB_MASTER_PORT} > > - if [ "x${SB_MASTER_PROTO}" = xtcp ]; then > - set $@ --db-sb-create-insecure-remote=yes > + else > If LISTEN_ON_MASTER_IP_ONLY is yes (which is the default case), the below 2 lines make all the servers to listen on master ip. As per the discussion we had in the previous patch set, we want to listen on the master ip only the master node. But I think it is fine for now and we can address this in a separate patch. + set $@ --db-nb-addr=${MASTER_IP} --db-nb-port=${NB_MASTER_PORT} > + set $@ --db-sb-addr=${MASTER_IP} --db-sb-port=${SB_MASTER_PORT} > fi > > if [ "x${present_master}" = x ]; then > @@ -313,15 +329,33 @@ ovsdb_server_start() { > # Force all copies to come up as slaves by pointing them into > # space and let pacemaker pick one to promote: > # > + if [ "x${NB_MASTER_PROTO}" = xtcp ]; then > + set $@ --db-nb-create-insecure-remote=yes > + fi > + > + if [ "x${SB_MASTER_PROTO}" = xtcp ]; then > + set $@ --db-sb-create-insecure-remote=yes > + fi > set $@ --db-nb-sync-from-addr=${INVALID_IP_ADDRESS} > --db-sb-sync-from-addr=${INVALID_IP_ADDRESS} > > elif [ ${present_master} != ${host_name} ]; then > + # TODO: for using LB vip, need to test for ssl. > + if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xyes ]; then > + if [ "x${NB_MASTER_PROTO}" = xtcp ]; then > + set $@ --db-nb-create-insecure-remote=yes > + fi > + > + if [ "x${SB_MASTER_PROTO}" = xtcp ]; then > + set $@ --db-sb-create-insecure-remote=yes > + fi > + fi > # An existing master is active, connect to it > set $@ --db-nb-sync-from-addr=${MASTER_IP} > --db-sb-sync-from-addr=${MASTER_IP} > set $@ --db-nb-sync-from-port=${NB_MASTER_PORT} > set $@ --db-nb-sync-from-proto=${NB_MASTER_PROTO} > set $@ --db-sb-sync-from-port=${SB_MASTER_PORT} > set $@ --db-sb-sync-from-proto=${SB_MASTER_PROTO} > + > fi > > $@ start_ovsdb > @@ -416,6 +450,12 @@ ovsdb_server_promote() { > ;; > esac > > + # Restart ovs so that new master can listen on tcp port. > + # TODO make sure to do more testing as unifying ovs restart both LB > and > + # pacemaker VIP resource do not break existing pacemkaer vip > functionality > + # for managing ovndbs. > If LISTEN_ON_MASTER_IP_ONLY is yes, then the slave nodes would have started with --remote= tcp:MASTER_IP:PORT. So I think it is not required to restart the ovsdb-servers for this case. Can you please update the patch with the below if condition to restart only for the other case. I tested with the below changes and it worked fine for me. Other then this change, the rest of the patch looks good to me. if [ "x${LISTEN_ON_MASTER_IP_ONLY}" = xno ]; then ${OVN_CTL} stop_ovsdb ovsdb_server_start fi Thanks Numan + ${OVN_CTL} stop_ovsdb > + ovsdb_server_start ${OVN_CTL} promote_ovnnb > ${OVN_CTL} promote_ovnsb > > -- > 1.9.1 > > _______________________________________________ > dev mailing list > d...@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev > _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
