Hi Numan, thanks for the new version. Did you (or someone else) re-test this with the changes that I made? I did not test them myself, so I have low confidence in them. The upgrade_cluster() function especially needs testing.
Thanks, Ben. On Wed, Apr 04, 2018 at 09:56:54PM +0530, nusid...@redhat.com wrote: > From: Numan Siddique <nusid...@redhat.com> > > This patch adds the options to start clustered OVN db servers in ovn-ctl. > To support this, following options are added - > '--db-(nb/sb)-cluster-local-addr', > '--db-(nb/sb)-cluster-local-port', '--db-(nb/sb)-cluster-local-proto', > '--db-(nb/sb)-cluster-remote-addr', '--db-(nb/sb)-cluster-remote-port' and > '--db-(nb/sb)-cluster-remote-proto'. > > If only '--db-(nb/sb)-cluster-local-addr' is defined then clustered db is > created > (using ovsdb-tool create-cluster). If both are defined, then the db is added > to > the cluster (using ovsdb-tool join-cluster) > > This patch also adds the support to configure ovn-northd to point to all the > servers > in the cluster using the options - '--ovn-northd-nb-db' and > 'ovn-northd-sb-db'. > > Presently this patch doesn't handle the schema update scenario when > restarting the > clustered ovsdb-servers. This will be handled in a separate patch. > > The initial versions of these commands are tested by Aliasgar <aginwala > <aginw...@asu.edu> > and the discussion on this can be found here - > https://mail.openvswitch.org/pipermail/ovs-discuss/2018-March/046470.html > > (There are 4 checkpatch warnings 'Line length is >79-characters long' in > ovn-ctl.8.xml > which I couldn't resolve without losing proper rendering when "man ovn-ctl" > is run.) > > Co-authored-by: Ben Pfaff <b...@ovn.org> > Signed-off-by: Numan Siddique <nusid...@redhat.com> > Signed-off-by: Ben Pfaff <b...@ovn.org> > --- > ovn/utilities/ovn-ctl | 191 > +++++++++++++++++++++++++++++--------------- > ovn/utilities/ovn-ctl.8.xml | 71 ++++++++++++++++ > utilities/ovs-lib.in | 62 ++++++++++++-- > 3 files changed, 255 insertions(+), 69 deletions(-) > > diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl > index dc0c26159..25dda524c 100755 > --- a/ovn/utilities/ovn-ctl > +++ b/ovn/utilities/ovn-ctl > @@ -93,84 +93,103 @@ promote_ovnsb() { > ovs-appctl -t $rundir/ovnsb_db.ctl > ovsdb-server/disconnect-active-ovsdb-server > } > > -start_nb_ovsdb() { > - # Check and eventually start ovsdb-server for Northbound DB > - if ! pidfile_is_running $DB_NB_PID; then > - upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null > +start_ovsdb__() { > + local DB=$1 db=$2 schema_name=$3 table_name=$4 > + eval local pid=\$DB_${DB}_PID > + eval local cluster_local_addr=\$DB_${DB}_CLUSTER_LOCAL_ADDR > + eval local cluster_local_port=\$DB_${DB}_CLUSTER_LOCAL_PORT > + eval local cluster_local_proto=\$DB_${DB}_CLUSTER_LOCAL_PROTO > + eval local cluster_remote_addr=\$DB_${DB}_CLUSTER_REMOTE_ADDR > + eval local cluster_remote_port=\$DB_${DB}_CLUSTER_REMOTE_PORT > + eval local cluster_remote_proto=\$DB_${DB}_CLUSTER_REMOTE_PROTO > + eval local sync_from_proto=\$DB_${DB}_SYNC_FROM_PROTO > + eval local sync_from_addr=\$DB_${DB}_SYNC_FROM_ADDR > + eval local sync_from_port=\$DB_${DB}_SYNC_FROM_PORT > + eval local file=\$DB_${DB}_FILE > + eval local schema=\$DB_${DB}_SCHEMA > + eval local logfile=\$OVN_${DB}_LOGFILE > + eval local log=\$OVN_${DB}_LOG > + eval local sock=\$DB_${DB}_SOCK > + eval local detach=\$DB_${DB}_DETACH > + eval local create_insecure_remote=\$DB_${DB}_CREATE_INSECURE_REMOTE > + eval local port=\$DB_${DB}_PORT > + eval local addr=\$DB_${DB}_ADDR > + eval local active_conf_file=\$ovn${db}_active_conf_file > + > + # Check and eventually start ovsdb-server for DB > + if pidfile_is_running $pid; then > + return > + fi > > - set ovsdb-server > + if test ! -z "$cluster_local_addr"; then > + mode=cluster > + elif test ! -z "$sync_from_addr"; then > + mode=active_passive > + echo "$sync_from_proto:$sync_from_addr:\ > +$sync_from_port" > $active_conf_file > + else > + mode=standalone > + fi > > - if test X"$DB_NB_DETACH" != Xno; then > - set "$@" --detach --monitor > + if test $mode = cluster; then > + local local=$cluster_local_proto:$cluster_local_addr:\ > +$cluster_local_port > + local remote=$cluster_remote_proto:$cluster_remote_addr:\ > +$cluster_remote_port > + if test -n "$cluster_remote_addr"; then > + join_cluster "$file" "$schema_name" "$local" "$remote" > else > - set exec "$@" > + create_cluster "$file" "$schema" "$local" > fi > + else > + upgrade_db "$file" "$schema" > + fi > > - set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE > - set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID > - set "$@" --remote=db:OVN_Northbound,NB_Global,connections > - set "$@" --unixctl=ovnnb_db.ctl > - set "$@" --private-key=db:OVN_Northbound,SSL,private_key > - set "$@" --certificate=db:OVN_Northbound,SSL,certificate > - set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert > - set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols > - set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers > - > - if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then > - set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR > - fi > + set ovsdb-server > + set "$@" $log --log-file=$logfile > + set "$@" --remote=punix:$sock --pidfile=$pid > + set "$@" --unixctl=ovn${db}_db.ctl > > - if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then > - echo > "$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" > > $ovnnb_active_conf_file > - fi > + if test X"$detach" != Xno; then > + set "$@" --detach --monitor > + else > + set exec "$@" > + fi > > - if test -e $ovnnb_active_conf_file; then > - set "$@" --sync-from=`cat $ovnnb_active_conf_file` > - fi > + set "$@" --remote=db:$schema_name,$table_name,connections > + set "$@" --private-key=db:$schema_name,SSL,private_key > + set "$@" --certificate=db:$schema_name,SSL,certificate > + set "$@" --ca-cert=db:$schema_name,SSL,ca_cert > + set "$@" --ssl-protocols=db:$schema_name,SSL,ssl_protocols > + set "$@" --ssl-ciphers=db:$schema_name,SSL,ssl_ciphers > > - $@ $DB_NB_FILE > - ovn-nbctl init > + if test X"$create_insecure_remote" = Xyes; then > + set "$@" --remote=ptcp:$port:$addr > fi > -} > > -start_sb_ovsdb() { > - # Check and eventually start ovsdb-server for Southbound DB > - if ! pidfile_is_running $DB_SB_PID; then > - upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null > - > - set ovsdb-server > + if test $mode = active_passive; then > + set "$@" --sync-from=`cat $active_conf_file` > + fi > > - if test X"$DB_SB_DETACH" != Xno; then > - set "$@" --detach --monitor > - else > - set exec "$@" > - fi > + "$@" "$file" > > - set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE > - set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID > - set "$@" --remote=db:OVN_Southbound,SB_Global,connections > - set "$@" --unixctl=ovnsb_db.ctl > - set "$@" --private-key=db:OVN_Southbound,SSL,private_key > - set "$@" --certificate=db:OVN_Southbound,SSL,certificate > - set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert > - set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols > - set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers > - > - if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then > - set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR > - fi > + # Initialize the database if it's running standalone, > + # active-passive, or is the first server in a cluster. > + if test -z "$cluster_remote_addr"; then > + ovn-nbctl init > + fi > > - if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then > - echo > "$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" > > $ovnsb_active_conf_file > - fi > + if test $mode = cluster; then > + upgrade_cluster "$schema" "unix:$sock" > + fi > +} > > - if test -e $ovnsb_active_conf_file; then > - set "$@" --sync-from=`cat $ovnsb_active_conf_file` > - fi > +start_nb_ovsdb() { > + start_ovsdb__ NB nb OVN_Northbound NB_Global > +} > > - $@ $DB_SB_FILE > - ovn-sbctl init > - fi > +start_sb_ovsdb() { > + start_ovsdb__ SB sb OVN_Southbound SB_Global > } > > start_ovsdb () { > @@ -236,7 +255,8 @@ start_northd () { > exit > fi > fi > - ovn_northd_params="--ovnnb-db=unix:$DB_NB_SOCK > --ovnsb-db=unix:$DB_SB_SOCK" > + ovn_northd_params="--ovnnb-db=$OVN_NORTHD_NB_DB \ > + --ovnsb-db=$OVN_NORTHD_SB_DB" > else > ovn_northd_params="`cat $ovn_northd_db_conf_file`" > fi > @@ -406,6 +426,23 @@ set_defaults () { > > DB_NB_DETACH="yes" > DB_SB_DETACH="yes" > + > + DB_NB_CLUSTER_LOCAL_ADDR="" > + DB_NB_CLUSTER_LOCAL_PROTO="tcp" > + DB_NB_CLUSTER_LOCAL_PORT=6643 > + DB_NB_CLUSTER_REMOTE_ADDR="" > + DB_NB_CLUSTER_REMOTE_PROTO="tcp" > + DB_NB_CLUSTER_REMOTE_PORT=6643 > + > + DB_SB_CLUSTER_LOCAL_ADDR="" > + DB_SB_CLUSTER_LOCAL_PROTO="tcp" > + DB_SB_CLUSTER_LOCAL_PORT=6644 > + DB_SB_CLUSTER_REMOTE_ADDR="" > + DB_SB_CLUSTER_REMOTE_PROTO="tcp" > + DB_SB_CLUSTER_REMOTE_PORT=6644 > + > + OVN_NORTHD_NB_DB="unix:$DB_NB_SOCK" > + OVN_NORTHD_SB_DB="unix:$DB_SB_SOCK" > } > > set_option () { > @@ -494,6 +531,32 @@ File location options: > --db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default: > $DB_SB_SYNC_FROM_PORT) > --db-sb-sync-from-proto=PROTO OVN Southbound active db transport (default: > $DB_SB_SYNC_FROM_PROTO) > --db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote > (default: $DB_SB_CREATE_INSECURE_REMOTE) > + --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \ > + (default: $DB_NB_CLUSTER_LOCAL_ADDR) > + --db-nb-cluster-local-port=PORT OVN_Northbound cluster local tcp port \ > + (default: $DB_NB_CLUSTER_LOCAL_PORT) > + --db-nb-cluster-local-proto=PROTO OVN_Northbound cluster local db > transport \ > + (default: $DB_NB_CLUSTER_LOCAL_PROTO) > + --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \ > + (default: $DB_NB_CLUSTER_REMOTE_ADDR) > + --db-nb-cluster-remote-port=PORT OVN_Northbound cluster remote tcp port \ > + (default: $DB_NB_CLUSTER_REMOTE_PORT) > + --db-nb-cluster-remote-proto=PROTO OVN_Northbound cluster remote db \ > + transport (default: $DB_NB_CLUSTER_REMOTE_PROTO) > + --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \ > + (default: $DB_SB_CLUSTER_LOCAL_ADDR) > + --db-sb-cluster-local-port=PORT OVN_Southbound cluster local tcp port \ > + (default: $DB_SB_CLUSTER_LOCAL_PORT) > + --db-sb-cluster-local-proto=PROTO OVN_Southbound cluster local db > transport \ > + (default: $DB_SB_CLUSTER_LOCAL_PROTO) > + --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \ > + (default: $DB_SB_CLUSTER_REMOTE_ADDR) > + --db-sb-cluster-remote-port=PORT OVN_Southbound cluster remote tcp port \ > + (default: $DB_SB_CLUSTER_REMOTE_PORT) > + --db-sb-cluster-remote-proto=PROTO OVN_Southbound cluster remote db \ > + transport (default: $DB_SB_CLUSTER_REMOTE_PROTO) > + --ovn-northd-nb-db=NB DB address(es) (default: $OVN_NORTHD_NB_DB) > + --ovn-northd-sb-db=SB DB address(es) (default: $OVN_NORTHD_SB_DB) > > Default directories with "configure" option and environment variable > override: > logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR) > diff --git a/ovn/utilities/ovn-ctl.8.xml b/ovn/utilities/ovn-ctl.8.xml > index 40defc9ec..02235fe1e 100644 > --- a/ovn/utilities/ovn-ctl.8.xml > +++ b/ovn/utilities/ovn-ctl.8.xml > @@ -66,6 +66,31 @@ > <p><code>--db-sb-sync-from-addr=<var>IP ADDRESS</var></code></p> > <p><code>--db-sb-sync-from-port=<var>PORT NUMBER</var></code></p> > <p><code>--db-sb-sync-from-proto=<var>PROTO</var></code></p> > + <p> > + <code> > + --ovn-northd-nb-db=<var>PROTO</var>:<var>IP ADDRESS</var>: > + <var>PORT</var>.. > + </code> > + </p> > + <p> > + <code> > + --ovn-northd-sb-db=<var>PROTO</var>:<var>IP ADDRESS</var>: > + <var>PORT</var>.. > + </code> > + </p> > + <h1> Clustering options </h1> > + <p><code>--db-nb-cluster-local-addr=<var>IP ADDRESS</var></code></p> > + <p><code>--db-nb-cluster-local-port=<var>PORT NUMBER</var></code></p> > + <p><code>--db-nb-cluster-local-proto=<var>PROTO > (tcp/ssl)</var></code></p> > + <p><code>--db-nb-cluster-remote-addr=<var>IP ADDRESS</var></code></p> > + <p><code>--db-nb-cluster-remote-port=<var>PORT NUMBER</var></code></p> > + <p><code>--db-nb-cluster-remote-proto=<var>PROTO > (tcp/ssl)</var></code></p> > + <p><code>--db-sb-cluster-local-addr=<var>IP ADDRESS</var></code></p> > + <p><code>--db-sb-cluster-local-port=<var>PORT NUMBER</var></code></p> > + <p><code>--db-sb-cluster-local-proto=<var>PROTO > (tcp/ssl)</var></code></p> > + <p><code>--db-sb-cluster-remote-addr=<var>IP ADDRESS</var></code></p> > + <p><code>--db-sb-cluster-remote-port=<var>PORT NUMBER</var></code></p> > + <p><code>--db-sb-cluster-remote-proto=<var>PROTO > (tcp/ssl)</var></code></p> > > <h1>Configuration files</h1> > <p>Following are the optional configuration files. If present, it should > be located in the etc dir</p> > @@ -125,4 +150,50 @@ > <p><code># ovn-ctl promote_ovnsb</code></p> > <p><code># ovn-ctl --db-nb-sync-from-addr=x.x.x.x > --db-nb-sync-from-port=6641 demote_ovnnb</code></p> > <p><code># ovn-ctl --db-sb-sync-from-addr=x.x.x.x > --db-sb-sync-from-port=6642 demote_ovnsb</code></p> > + > + <h2>Creating a clustered db on 3 nodes with IPs x.x.x.x, y.y.y.y and > z.z.z.z</h2> > + <h3>Starting OVN ovsdb servers and ovn-northd on the node with IP > x.x.x.x</h3> > + <p> > + <code> > + # ovn-ctl --db-nb-addr=x.x.x.x --db-nb-create-insecure-remote=yes > + --db-sb-addr=x.x.x.x --db-sb-create-insecure-remote=yes > + --db-nb-cluster-local-addr=x.x.x.x > + --db-sb-cluster-local-addr=x.x.x.x > + --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641 > + --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642 > + start_northd > + </code> > + </p> > + > + <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP > y.y.y.y and joining the cluster started at x.x.x.x</h3> > + <p> > + <code> > + # ovn-ctl --db-nb-addr=y.y.y.y --db-nb-create-insecure-remote=yes > + --db-sb-addr=y.y.y.y --db-sb-create-insecure-remote=yes > + --db-nb-cluster-local-addr=y.y.y.y > + --db-sb-cluster-local-addr=y.y.y.y > + --db-nb-cluster-remote-addr=x.x.x.x > + --db-sb-cluster-remote-addr=x.x.x.x > + --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641 > + --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642 > + start_northd > + </code> > + </p> > + > + <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP > z.z.z.z and joining the cluster started at x.x.x.x</h3> > + <p> > + <code> > + # ovn-ctl --db-nb-addr=z.z.z.z > + --db-nb-create-insecure-remote=yes > + --db-nb-cluster-local-addr=z.z.z.z > + --db-sb-addr=z.z.z.z > + --db-sb-create-insecure-remote=yes > + --db-sb-cluster-local-addr=z.z.z.z > + --db-nb-cluster-remote-addr=x.x.x.x > + --db-sb-cluster-remote-addr=x.x.x.x > + > --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641 > + > --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642 > + start_northd > + </code> > + </p> > </manpage> > diff --git a/utilities/ovs-lib.in b/utilities/ovs-lib.in > index 4b08f0f53..4dc315118 100644 > --- a/utilities/ovs-lib.in > +++ b/utilities/ovs-lib.in > @@ -401,6 +401,14 @@ create_db () { > action "Creating empty database $DB_FILE" ovsdb_tool create "$DB_FILE" > "$DB_SCHEMA" > } > > +backup_db () { > + # Back up the old version. > + version=`ovsdb_tool db-version "$DB_FILE"` > + cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'` > + backup=$DB_FILE.backup$version-$cksum > + action "Backing up database to $backup" cp "$DB_FILE" "$backup" || > return 1 > +} > + > upgrade_db () { > DB_FILE="$1" > DB_SCHEMA="$2" > @@ -411,11 +419,7 @@ upgrade_db () { > install_dir `dirname $DB_FILE` > create_db "$DB_FILE" "$DB_SCHEMA" > elif test X"`ovsdb_tool needs-conversion "$DB_FILE" "$DB_SCHEMA"`" = > Xyes; then > - # Back up the old version. > - version=`ovsdb_tool db-version "$DB_FILE"` > - cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'` > - backup=$DB_FILE.backup$version-$cksum > - action "Backing up database to $backup" cp "$DB_FILE" "$backup" || > return 1 > + backup_db || return 1 > > # Compact database. This is important if the old schema did not > enable > # garbage collection (i.e. if it did not have any tables with > "isRoot": > @@ -443,6 +447,54 @@ upgrade_db () { > fi > } > > +upgrade_cluster () { > + local DB_SCHEMA=$1 DB_SERVER=$2 > + local schema_name=$(ovsdb-tool schema-name $1) || return 1 > + > + action "Waiting for $schema_name to come up" ovsdb-client -t 30 wait > "$DB_SERVER" "$schema_name" connected || return $? > + local db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" > "$schema_name") || return $? > + local target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") || return > $? > + > + if ovsdb-tool compare-versions "$db_version" == "$target_version"; then > + : > + elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; > then > + log_warning_msg "Database $schema_name has newer schema version > ($db_version) than our local schema ($target_version), possibly an upgrade is > partially complete?" > + else > + action "Upgrading database $schema_name from schema version > $db_version to $target_version" ovsdb-client -t 30 convert "$DB_SERVER" > "$DB_SCHEMA" > + fi > +} > + > +create_cluster () { > + DB_FILE="$1" > + DB_SCHEMA="$2" > + LOCAL_ADDR="$3" > + > + if test ! -e "$DB_FILE"; then > + action "Creating cluster database $DB_FILE" ovsdb_tool > create-cluster "$DB_FILE" "$DB_SCHEMA" "$LOCAL_ADDR" > + elif ovsdb_tool db-is-standalone "$DB_FILE"; then > + # Convert standalone database to clustered. > + backup_db || return 1 > + action "Creating cluster database $DB_FILE from existing one" \ > + ovsdb_tool create-cluster "$DB_FILE" "$backup" "$LOCAL_ADDR" > + fi > +} > + > +join_cluster() { > + DB_FILE="$1" > + SCHEMA_NAME="$2" > + LOCAL_ADDR="$3" > + REMOTE_ADDR="$4" > + > + if test ! -e "$DB_FILE"; then > + ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR" > "$REMOTE_ADDR" > + elif ovsdb_tool db-is-standalone "$DB_FILE"; then > + # Backup standalone database and join cluster. > + backup_db || return 1 > + action "Joining $DB_FILE to cluster" \ > + ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" > "$LOCAL_ADDR" > + fi > +} > + > ovs_vsctl () { > ovs-vsctl --no-wait "$@" > } > -- > 2.14.3 > _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
