From: Numan Siddique <nusid...@redhat.com>

This patch adds the options to start clustered OVN db servers in ovn-ctl.
To support this, following options are added - 
'--db-(nb/sb)-cluster-local-addr',
'--db-(nb/sb)-cluster-local-port', '--db-(nb/sb)-cluster-local-proto',
'--db-(nb/sb)-cluster-remote-addr', '--db-(nb/sb)-cluster-remote-port' and
'--db-(nb/sb)-cluster-remote-proto'.

If only '--db-(nb/sb)-cluster-local-addr' is defined then clustered db is 
created
(using ovsdb-tool create-cluster). If both are defined, then the db is added to
the cluster (using ovsdb-tool join-cluster)

This patch also adds the support to configure ovn-northd to point to all the 
servers
in the cluster using the options - '--ovn-northd-nb-db' and 'ovn-northd-sb-db'.

Presently this patch doesn't handle the schema update scenario when restarting 
the
clustered ovsdb-servers. This will be handled in a separate patch.

The initial versions of these commands are tested by Aliasgar <aginwala 
<aginw...@asu.edu>
and the discussion on this can be found here -
https://mail.openvswitch.org/pipermail/ovs-discuss/2018-March/046470.html

(There are 4 checkpatch warnings 'Line length is >79-characters long' in 
ovn-ctl.8.xml
which I couldn't resolve without losing proper rendering when "man ovn-ctl" is 
run.)

Co-authored-by: Ben Pfaff <b...@ovn.org>
Signed-off-by: Numan Siddique <nusid...@redhat.com>
Signed-off-by: Ben Pfaff <b...@ovn.org>
---
 ovn/utilities/ovn-ctl       | 191 +++++++++++++++++++++++++++++---------------
 ovn/utilities/ovn-ctl.8.xml |  71 ++++++++++++++++
 utilities/ovs-lib.in        |  62 ++++++++++++--
 3 files changed, 255 insertions(+), 69 deletions(-)

diff --git a/ovn/utilities/ovn-ctl b/ovn/utilities/ovn-ctl
index dc0c26159..25dda524c 100755
--- a/ovn/utilities/ovn-ctl
+++ b/ovn/utilities/ovn-ctl
@@ -93,84 +93,103 @@ promote_ovnsb() {
     ovs-appctl -t $rundir/ovnsb_db.ctl 
ovsdb-server/disconnect-active-ovsdb-server
 }
 
-start_nb_ovsdb() {
-    # Check and eventually start ovsdb-server for Northbound DB
-    if ! pidfile_is_running $DB_NB_PID; then
-        upgrade_db "$DB_NB_FILE" "$DB_NB_SCHEMA" 1>/dev/null 2>/dev/null
+start_ovsdb__() {
+    local DB=$1 db=$2 schema_name=$3 table_name=$4
+    eval local pid=\$DB_${DB}_PID
+    eval local cluster_local_addr=\$DB_${DB}_CLUSTER_LOCAL_ADDR
+    eval local cluster_local_port=\$DB_${DB}_CLUSTER_LOCAL_PORT
+    eval local cluster_local_proto=\$DB_${DB}_CLUSTER_LOCAL_PROTO
+    eval local cluster_remote_addr=\$DB_${DB}_CLUSTER_REMOTE_ADDR
+    eval local cluster_remote_port=\$DB_${DB}_CLUSTER_REMOTE_PORT
+    eval local cluster_remote_proto=\$DB_${DB}_CLUSTER_REMOTE_PROTO
+    eval local sync_from_proto=\$DB_${DB}_SYNC_FROM_PROTO
+    eval local sync_from_addr=\$DB_${DB}_SYNC_FROM_ADDR
+    eval local sync_from_port=\$DB_${DB}_SYNC_FROM_PORT
+    eval local file=\$DB_${DB}_FILE
+    eval local schema=\$DB_${DB}_SCHEMA
+    eval local logfile=\$OVN_${DB}_LOGFILE
+    eval local log=\$OVN_${DB}_LOG
+    eval local sock=\$DB_${DB}_SOCK
+    eval local detach=\$DB_${DB}_DETACH
+    eval local create_insecure_remote=\$DB_${DB}_CREATE_INSECURE_REMOTE
+    eval local port=\$DB_${DB}_PORT
+    eval local addr=\$DB_${DB}_ADDR
+    eval local active_conf_file=\$ovn${db}_active_conf_file
+
+    # Check and eventually start ovsdb-server for DB
+    if pidfile_is_running $pid; then
+        return
+    fi
 
-        set ovsdb-server
+    if test ! -z "$cluster_local_addr"; then
+        mode=cluster
+    elif test ! -z "$sync_from_addr"; then
+        mode=active_passive
+        echo "$sync_from_proto:$sync_from_addr:\
+$sync_from_port" > $active_conf_file
+    else
+        mode=standalone
+    fi
 
-        if test X"$DB_NB_DETACH" != Xno; then
-            set "$@" --detach --monitor
+    if test $mode = cluster; then
+        local local=$cluster_local_proto:$cluster_local_addr:\
+$cluster_local_port
+        local remote=$cluster_remote_proto:$cluster_remote_addr:\
+$cluster_remote_port
+        if test -n "$cluster_remote_addr"; then
+            join_cluster "$file" "$schema_name" "$local" "$remote"
         else
-            set exec "$@"
+            create_cluster "$file" "$schema" "$local"
         fi
+    else
+        upgrade_db "$file" "$schema"
+    fi
 
-        set "$@" $OVN_NB_LOG --log-file=$OVN_NB_LOGFILE
-        set "$@" --remote=punix:$DB_NB_SOCK --pidfile=$DB_NB_PID
-        set "$@" --remote=db:OVN_Northbound,NB_Global,connections
-        set "$@" --unixctl=ovnnb_db.ctl
-        set "$@" --private-key=db:OVN_Northbound,SSL,private_key
-        set "$@" --certificate=db:OVN_Northbound,SSL,certificate
-        set "$@" --ca-cert=db:OVN_Northbound,SSL,ca_cert
-        set "$@" --ssl-protocols=db:OVN_Northbound,SSL,ssl_protocols
-        set "$@" --ssl-ciphers=db:OVN_Northbound,SSL,ssl_ciphers
-
-        if test X"$DB_NB_CREATE_INSECURE_REMOTE" = Xyes; then
-            set "$@" --remote=ptcp:$DB_NB_PORT:$DB_NB_ADDR
-        fi
+    set ovsdb-server
+    set "$@" $log --log-file=$logfile
+    set "$@" --remote=punix:$sock --pidfile=$pid
+    set "$@" --unixctl=ovn${db}_db.ctl
 
-        if test ! -z "$DB_NB_SYNC_FROM_ADDR"; then
-            echo 
"$DB_NB_SYNC_FROM_PROTO:$DB_NB_SYNC_FROM_ADDR:$DB_NB_SYNC_FROM_PORT" > 
$ovnnb_active_conf_file
-        fi
+    if test X"$detach" != Xno; then
+        set "$@" --detach --monitor
+    else
+        set exec "$@"
+    fi
 
-        if test -e $ovnnb_active_conf_file; then
-            set "$@" --sync-from=`cat $ovnnb_active_conf_file`
-        fi
+    set "$@" --remote=db:$schema_name,$table_name,connections
+    set "$@" --private-key=db:$schema_name,SSL,private_key
+    set "$@" --certificate=db:$schema_name,SSL,certificate
+    set "$@" --ca-cert=db:$schema_name,SSL,ca_cert
+    set "$@" --ssl-protocols=db:$schema_name,SSL,ssl_protocols
+    set "$@" --ssl-ciphers=db:$schema_name,SSL,ssl_ciphers
 
-        $@ $DB_NB_FILE
-        ovn-nbctl init
+    if test X"$create_insecure_remote" = Xyes; then
+        set "$@" --remote=ptcp:$port:$addr
     fi
-}
 
-start_sb_ovsdb() {
-    # Check and eventually start ovsdb-server for Southbound DB
-    if ! pidfile_is_running $DB_SB_PID; then
-        upgrade_db "$DB_SB_FILE" "$DB_SB_SCHEMA" 1>/dev/null 2>/dev/null
-
-        set ovsdb-server
+    if test $mode = active_passive; then
+        set "$@" --sync-from=`cat $active_conf_file`
+    fi
 
-        if test X"$DB_SB_DETACH" != Xno; then
-            set "$@" --detach --monitor
-        else
-            set exec "$@"
-        fi
+    "$@" "$file"
 
-        set "$@" $OVN_SB_LOG --log-file=$OVN_SB_LOGFILE
-        set "$@" --remote=punix:$DB_SB_SOCK --pidfile=$DB_SB_PID
-        set "$@" --remote=db:OVN_Southbound,SB_Global,connections
-        set "$@" --unixctl=ovnsb_db.ctl
-        set "$@" --private-key=db:OVN_Southbound,SSL,private_key
-        set "$@" --certificate=db:OVN_Southbound,SSL,certificate
-        set "$@" --ca-cert=db:OVN_Southbound,SSL,ca_cert
-        set "$@" --ssl-protocols=db:OVN_Southbound,SSL,ssl_protocols
-        set "$@" --ssl-ciphers=db:OVN_Southbound,SSL,ssl_ciphers
-
-        if test X"$DB_SB_CREATE_INSECURE_REMOTE" = Xyes; then
-            set "$@" --remote=ptcp:$DB_SB_PORT:$DB_SB_ADDR
-        fi
+    # Initialize the database if it's running standalone,
+    # active-passive, or is the first server in a cluster.
+    if test -z "$cluster_remote_addr"; then
+        ovn-nbctl init
+    fi
 
-        if test ! -z "$DB_SB_SYNC_FROM_ADDR"; then
-            echo 
"$DB_SB_SYNC_FROM_PROTO:$DB_SB_SYNC_FROM_ADDR:$DB_SB_SYNC_FROM_PORT" > 
$ovnsb_active_conf_file
-        fi
+    if test $mode = cluster; then
+        upgrade_cluster "$schema" "unix:$sock"
+    fi
+}
 
-        if test -e $ovnsb_active_conf_file; then
-            set "$@" --sync-from=`cat $ovnsb_active_conf_file`
-        fi
+start_nb_ovsdb() {
+    start_ovsdb__ NB nb OVN_Northbound NB_Global
+}
 
-        $@ $DB_SB_FILE
-        ovn-sbctl init
-    fi
+start_sb_ovsdb() {
+    start_ovsdb__ SB sb OVN_Southbound SB_Global
 }
 
 start_ovsdb () {
@@ -236,7 +255,8 @@ start_northd () {
                 exit
             fi
         fi
-        ovn_northd_params="--ovnnb-db=unix:$DB_NB_SOCK 
--ovnsb-db=unix:$DB_SB_SOCK"
+        ovn_northd_params="--ovnnb-db=$OVN_NORTHD_NB_DB \
+        --ovnsb-db=$OVN_NORTHD_SB_DB"
     else
         ovn_northd_params="`cat $ovn_northd_db_conf_file`"
     fi
@@ -406,6 +426,23 @@ set_defaults () {
 
     DB_NB_DETACH="yes"
     DB_SB_DETACH="yes"
+
+    DB_NB_CLUSTER_LOCAL_ADDR=""
+    DB_NB_CLUSTER_LOCAL_PROTO="tcp"
+    DB_NB_CLUSTER_LOCAL_PORT=6643
+    DB_NB_CLUSTER_REMOTE_ADDR=""
+    DB_NB_CLUSTER_REMOTE_PROTO="tcp"
+    DB_NB_CLUSTER_REMOTE_PORT=6643
+
+    DB_SB_CLUSTER_LOCAL_ADDR=""
+    DB_SB_CLUSTER_LOCAL_PROTO="tcp"
+    DB_SB_CLUSTER_LOCAL_PORT=6644
+    DB_SB_CLUSTER_REMOTE_ADDR=""
+    DB_SB_CLUSTER_REMOTE_PROTO="tcp"
+    DB_SB_CLUSTER_REMOTE_PORT=6644
+
+    OVN_NORTHD_NB_DB="unix:$DB_NB_SOCK"
+    OVN_NORTHD_SB_DB="unix:$DB_SB_SOCK"
 }
 
 set_option () {
@@ -494,6 +531,32 @@ File location options:
   --db-sb-sync-from-port=ADDR OVN Southbound active db tcp port (default: 
$DB_SB_SYNC_FROM_PORT)
   --db-sb-sync-from-proto=PROTO OVN Southbound active db transport (default: 
$DB_SB_SYNC_FROM_PROTO)
   --db-sb-create-insecure-remote=yes|no Create ptcp OVN Southbound remote 
(default: $DB_SB_CREATE_INSECURE_REMOTE)
+  --db-nb-cluster-local-addr=ADDR OVN_Northbound cluster local address \
+  (default: $DB_NB_CLUSTER_LOCAL_ADDR)
+  --db-nb-cluster-local-port=PORT OVN_Northbound cluster local tcp port \
+  (default: $DB_NB_CLUSTER_LOCAL_PORT)
+  --db-nb-cluster-local-proto=PROTO OVN_Northbound cluster local db transport \
+  (default: $DB_NB_CLUSTER_LOCAL_PROTO)
+  --db-nb-cluster-remote-addr=ADDR OVN_Northbound cluster remote address \
+  (default: $DB_NB_CLUSTER_REMOTE_ADDR)
+  --db-nb-cluster-remote-port=PORT OVN_Northbound cluster remote tcp port \
+  (default: $DB_NB_CLUSTER_REMOTE_PORT)
+  --db-nb-cluster-remote-proto=PROTO OVN_Northbound cluster remote db \
+  transport (default: $DB_NB_CLUSTER_REMOTE_PROTO)
+  --db-sb-cluster-local-addr=ADDR OVN_Southbound cluster local address \
+  (default: $DB_SB_CLUSTER_LOCAL_ADDR)
+  --db-sb-cluster-local-port=PORT OVN_Southbound cluster local tcp port \
+  (default: $DB_SB_CLUSTER_LOCAL_PORT)
+  --db-sb-cluster-local-proto=PROTO OVN_Southbound cluster local db transport \
+  (default: $DB_SB_CLUSTER_LOCAL_PROTO)
+  --db-sb-cluster-remote-addr=ADDR OVN_Southbound cluster remote address \
+  (default: $DB_SB_CLUSTER_REMOTE_ADDR)
+  --db-sb-cluster-remote-port=PORT OVN_Southbound cluster remote tcp port \
+  (default: $DB_SB_CLUSTER_REMOTE_PORT)
+  --db-sb-cluster-remote-proto=PROTO OVN_Southbound cluster remote db \
+  transport (default: $DB_SB_CLUSTER_REMOTE_PROTO)
+  --ovn-northd-nb-db=NB DB address(es) (default: $OVN_NORTHD_NB_DB)
+  --ovn-northd-sb-db=SB DB address(es) (default: $OVN_NORTHD_SB_DB)
 
 Default directories with "configure" option and environment variable override:
   logs: /usr/local/var/log/openvswitch (--with-logdir, OVS_LOGDIR)
diff --git a/ovn/utilities/ovn-ctl.8.xml b/ovn/utilities/ovn-ctl.8.xml
index 40defc9ec..02235fe1e 100644
--- a/ovn/utilities/ovn-ctl.8.xml
+++ b/ovn/utilities/ovn-ctl.8.xml
@@ -66,6 +66,31 @@
     <p><code>--db-sb-sync-from-addr=<var>IP ADDRESS</var></code></p>
     <p><code>--db-sb-sync-from-port=<var>PORT NUMBER</var></code></p>
     <p><code>--db-sb-sync-from-proto=<var>PROTO</var></code></p>
+    <p>
+      <code>
+        --ovn-northd-nb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+        <var>PORT</var>..
+      </code>
+    </p>
+    <p>
+      <code>
+        --ovn-northd-sb-db=<var>PROTO</var>:<var>IP ADDRESS</var>:
+        <var>PORT</var>..
+      </code>
+    </p>
+    <h1> Clustering options </h1>
+    <p><code>--db-nb-cluster-local-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-nb-cluster-local-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-nb-cluster-local-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-nb-cluster-remote-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-nb-cluster-remote-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-nb-cluster-remote-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-sb-cluster-local-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-sb-cluster-local-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-sb-cluster-local-proto=<var>PROTO (tcp/ssl)</var></code></p>
+    <p><code>--db-sb-cluster-remote-addr=<var>IP ADDRESS</var></code></p>
+    <p><code>--db-sb-cluster-remote-port=<var>PORT NUMBER</var></code></p>
+    <p><code>--db-sb-cluster-remote-proto=<var>PROTO (tcp/ssl)</var></code></p>
 
     <h1>Configuration files</h1>
     <p>Following are the optional configuration files. If present, it should 
be located in the etc dir</p>
@@ -125,4 +150,50 @@
     <p><code># ovn-ctl promote_ovnsb</code></p>
     <p><code># ovn-ctl --db-nb-sync-from-addr=x.x.x.x 
--db-nb-sync-from-port=6641 demote_ovnnb</code></p>
     <p><code># ovn-ctl --db-sb-sync-from-addr=x.x.x.x 
--db-sb-sync-from-port=6642 demote_ovnsb</code></p>
+
+    <h2>Creating a clustered db on 3 nodes with IPs x.x.x.x, y.y.y.y and 
z.z.z.z</h2>
+    <h3>Starting OVN ovsdb servers and ovn-northd on the node with IP 
x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=x.x.x.x --db-nb-create-insecure-remote=yes
+        --db-sb-addr=x.x.x.x --db-sb-create-insecure-remote=yes
+        --db-nb-cluster-local-addr=x.x.x.x
+        --db-sb-cluster-local-addr=x.x.x.x
+        --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+        --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+        start_northd
+      </code>
+    </p>
+
+    <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP y.y.y.y 
and joining the cluster started at x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=y.y.y.y --db-nb-create-insecure-remote=yes
+        --db-sb-addr=y.y.y.y --db-sb-create-insecure-remote=yes
+        --db-nb-cluster-local-addr=y.y.y.y
+        --db-sb-cluster-local-addr=y.y.y.y
+        --db-nb-cluster-remote-addr=x.x.x.x
+        --db-sb-cluster-remote-addr=x.x.x.x
+        --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+        --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+        start_northd
+      </code>
+    </p>
+
+    <h3>Starting OVN ovsdb-servers and ovn-northd on the node with IP z.z.z.z 
and joining the cluster started at x.x.x.x</h3>
+    <p>
+      <code>
+        # ovn-ctl --db-nb-addr=z.z.z.z
+          --db-nb-create-insecure-remote=yes
+          --db-nb-cluster-local-addr=z.z.z.z
+          --db-sb-addr=z.z.z.z
+          --db-sb-create-insecure-remote=yes
+          --db-sb-cluster-local-addr=z.z.z.z
+          --db-nb-cluster-remote-addr=x.x.x.x
+          --db-sb-cluster-remote-addr=x.x.x.x
+          --ovn-northd-nb-db=tcp:x.x.x.x:6641,tcp:y.y.y.y:6641,tcp:z.z.z.z:6641
+          --ovn-northd-sb-db=tcp:x.x.x.x:6642,tcp:y.y.y.y:6642,tcp:z.z.z.z:6642
+          start_northd
+      </code>
+    </p>
 </manpage>
diff --git a/utilities/ovs-lib.in b/utilities/ovs-lib.in
index 4b08f0f53..4dc315118 100644
--- a/utilities/ovs-lib.in
+++ b/utilities/ovs-lib.in
@@ -401,6 +401,14 @@ create_db () {
     action "Creating empty database $DB_FILE" ovsdb_tool create "$DB_FILE" 
"$DB_SCHEMA"
 }
 
+backup_db () {
+    # Back up the old version.
+    version=`ovsdb_tool db-version "$DB_FILE"`
+    cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
+    backup=$DB_FILE.backup$version-$cksum
+    action "Backing up database to $backup" cp "$DB_FILE" "$backup" || return 1
+}
+
 upgrade_db () {
     DB_FILE="$1"
     DB_SCHEMA="$2"
@@ -411,11 +419,7 @@ upgrade_db () {
         install_dir `dirname $DB_FILE`
         create_db "$DB_FILE" "$DB_SCHEMA"
     elif test X"`ovsdb_tool needs-conversion "$DB_FILE" "$DB_SCHEMA"`" = Xyes; 
then
-        # Back up the old version.
-        version=`ovsdb_tool db-version "$DB_FILE"`
-        cksum=`ovsdb_tool db-cksum "$DB_FILE" | awk '{print $1}'`
-        backup=$DB_FILE.backup$version-$cksum
-        action "Backing up database to $backup" cp "$DB_FILE" "$backup" || 
return 1
+        backup_db || return 1
 
         # Compact database.  This is important if the old schema did not enable
         # garbage collection (i.e. if it did not have any tables with "isRoot":
@@ -443,6 +447,54 @@ upgrade_db () {
     fi
 }
 
+upgrade_cluster () {
+    local DB_SCHEMA=$1 DB_SERVER=$2
+    local schema_name=$(ovsdb-tool schema-name $1) || return 1
+
+    action "Waiting for $schema_name to come up" ovsdb-client -t 30 wait 
"$DB_SERVER" "$schema_name" connected || return $?
+    local db_version=$(ovsdb-client -t 10 get-schema-version "$DB_SERVER" 
"$schema_name") || return $?
+    local target_version=$(ovsdb-tool schema-version "$DB_SCHEMA") || return $?
+
+    if ovsdb-tool compare-versions "$db_version" == "$target_version"; then
+        :
+    elif ovsdb-tool compare-versions "$db_version" ">" "$target_version"; then
+        log_warning_msg "Database $schema_name has newer schema version 
($db_version) than our local schema ($target_version), possibly an upgrade is 
partially complete?"
+    else
+        action "Upgrading database $schema_name from schema version 
$db_version to $target_version" ovsdb-client -t 30 convert "$DB_SERVER" 
"$DB_SCHEMA"
+    fi
+}
+
+create_cluster () {
+    DB_FILE="$1"
+    DB_SCHEMA="$2"
+    LOCAL_ADDR="$3"
+
+    if test ! -e "$DB_FILE"; then
+        action "Creating cluster database $DB_FILE" ovsdb_tool create-cluster 
"$DB_FILE" "$DB_SCHEMA" "$LOCAL_ADDR"
+    elif ovsdb_tool db-is-standalone "$DB_FILE"; then
+        # Convert standalone database to clustered.
+        backup_db || return 1
+        action "Creating cluster database $DB_FILE from existing one" \
+               ovsdb_tool create-cluster "$DB_FILE" "$backup" "$LOCAL_ADDR"
+    fi
+}
+
+join_cluster() {
+    DB_FILE="$1"
+    SCHEMA_NAME="$2"
+    LOCAL_ADDR="$3"
+    REMOTE_ADDR="$4"
+
+    if test ! -e "$DB_FILE"; then
+        ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR" 
"$REMOTE_ADDR"
+    elif ovsdb_tool db-is-standalone "$DB_FILE"; then
+        # Backup standalone database and join cluster.
+        backup_db || return 1
+        action "Joining $DB_FILE to cluster" \
+               ovsdb_tool join-cluster "$DB_FILE" "$SCHEMA_NAME" "$LOCAL_ADDR"
+    fi
+}
+
 ovs_vsctl () {
     ovs-vsctl --no-wait "$@"
 }
-- 
2.14.3

_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev

Reply via email to