Re: [ovs-dev] [PATCH] ofproto: Return error codes for Rule insertions
Hi Aaron, > Currently, rule_insert() API doesnot have return value. There are some > possible > scenarios where rule insertions can fail at run-time even though the static > checks during rule_construct() had passed previously. > > Some possible scenarios for failure of rule insertions: > **) Rule insertions can fail dynamically in Hybrid mode (both Openflow and > Normal switch functioning coexist) where the CAM space could get suddenly > filled up by Normal switch functioning and Openflow gets devoid of > available space. > **) Some deployments could have separate independent layers for HW rule > insertions and application layer to interact with OVS. HW layer > could face any dynamic issue during rule handling which application could > not have predicted/captured in rule-construction phase. > > Rule-insert errors for bundles are not handled in this pull-request. > Will be handled in upcoming pull request. > > Signed-off-by: Aravind Prasad S > --- >> Thanks for working on OVS. >> As noted, the patch has some submission errors. Please try submitting >> again with 'git send-email' to eliminate any potential mail client >> munging. Thanks a lot for the info and support. Submitted the patch with git -email. OVS group rocks :-). Very active in assistance and support too. Thanks, S. Aravind Prasad On Mon, Jul 9, 2018 at 5:09 AM Aaron Conole wrote: > Hi Arvind, > > Aravind Prasad writes: > > > Currently, rule_insert() API doesnot have return value. There are some > > possible > > scenarios where rule insertions can fail at run-time even though the > static > > checks during rule_construct() had passed previously. > > > > Some possible scenarios for failure of rule insertions: > > **) Rule insertions can fail dynamically in Hybrid mode (both Openflow > and > > Normal switch functioning coexist) where the CAM space could get suddenly > > filled up by Normal switch functioning and Openflow gets devoid of > > available space. > > **) Some deployments could have separate independent layers for HW rule > > insertions and application layer to interact with OVS. HW layer > > could face any dynamic issue during rule handling which application could > > not have predicted/captured in rule-construction phase. > > > > Rule-insert errors for bundles are not handled in this pull-request. > > Will be handled in upcoming pull request. > > > > Signed-off-by: Aravind Prasad S > > --- > > Thanks for working on OVS. > > As noted, the patch has some submission errors. Please try submitting > again with 'git send-email' to eliminate any potential mail client > munging. > > -Aaron > ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] [PATCH] Handle rule insert failures
--- ofproto/ofproto-dpif.c | 4 ++- ofproto/ofproto-provider.h | 6 ++-- ofproto/ofproto.c | 76 +- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c index ad1e8af..d1678ed 100644 --- a/ofproto/ofproto-dpif.c +++ b/ofproto/ofproto-dpif.c @@ -4443,7 +4443,7 @@ rule_construct(struct rule *rule_) return 0; } -static void +static enum ofperr rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) OVS_REQUIRES(ofproto_mutex) { @@ -4473,6 +4473,8 @@ rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) ovs_mutex_unlock(>stats_mutex); ovs_mutex_unlock(_rule->stats_mutex); } + +return 0; } static void diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h index 2b77b89..3f3d110 100644 --- a/ofproto/ofproto-provider.h +++ b/ofproto/ofproto-provider.h @@ -1297,8 +1297,8 @@ struct ofproto_class { struct rule *(*rule_alloc)(void); enum ofperr (*rule_construct)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; -void (*rule_insert)(struct rule *rule, struct rule *old_rule, -bool forward_counts) +enum ofperr (*rule_insert)(struct rule *rule, struct rule *old_rule, + bool forward_counts) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_delete)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_destruct)(struct rule *rule); @@ -1952,7 +1952,7 @@ enum ofperr ofproto_flow_mod_learn_start(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); void ofproto_flow_mod_learn_revert(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); -void ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, +enum ofperr ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, struct ofproto *orig_ofproto) OVS_REQUIRES(ofproto_mutex); void ofproto_add_flow(struct ofproto *, const struct match *, int priority, diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c index f946e27..81b2466 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -245,10 +245,12 @@ static void replace_rule_revert(struct ofproto *, struct rule *old_rule, struct rule *new_rule) OVS_REQUIRES(ofproto_mutex); -static void replace_rule_finish(struct ofproto *, struct ofproto_flow_mod *, -const struct openflow_mod_requester *, -struct rule *old_rule, struct rule *new_rule, -struct ovs_list *dead_cookies) +static enum ofperr replace_rule_finish(struct ofproto *, + struct ofproto_flow_mod *, + const struct openflow_mod_requester *, + struct rule *old_rule, + struct rule *new_rule, + struct ovs_list *dead_cookies) OVS_REQUIRES(ofproto_mutex); static void delete_flows__(struct rule_collection *, enum ofp_flow_removed_reason, @@ -270,7 +272,7 @@ static enum ofperr ofproto_flow_mod_start(struct ofproto *, static void ofproto_flow_mod_revert(struct ofproto *, struct ofproto_flow_mod *) OVS_REQUIRES(ofproto_mutex); -static void ofproto_flow_mod_finish(struct ofproto *, +static enum ofperr ofproto_flow_mod_finish(struct ofproto *, struct ofproto_flow_mod *, const struct openflow_mod_requester *) OVS_REQUIRES(ofproto_mutex); @@ -4855,7 +4857,7 @@ add_flow_revert(struct ofproto *ofproto, struct ofproto_flow_mod *ofm) } /* To be called after version bump. */ -static void +static enum ofperr add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, const struct openflow_mod_requester *req) OVS_REQUIRES(ofproto_mutex) @@ -4864,8 +4866,14 @@ add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, ? rule_collection_rules(>old_rules)[0] : NULL; struct rule *new_rule = rule_collection_rules(>new_rules)[0]; struct ovs_list dead_cookies = OVS_LIST_INITIALIZER(_cookies); +enum ofperr error = 0; + +error = replace_rule_finish(ofproto, ofm, req, old_rule, new_rule, +_cookies); +if (error) { +return error; +} -replace_rule_finish(ofproto, ofm, req, old_rule, new_rule, _cookies); learned_cookies_flush(ofproto, _cookies); if (old_rule) { @@ -4878,6 +4886,8 @@ add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, /* Send Vacancy Events for OF1.4+. */ send_table_status(ofproto, new_rule->table_id); } + +return error; }
[ovs-dev] [PATCH] ofproto: Return error codes for Rule insertions
Currently, rule_insert() API doesnot have return value. There are some possible scenarios where rule insertions can fail at run-time even though the static checks during rule_construct() had passed previously. Some possible scenarios for failure of rule insertions: **) Rule insertions can fail dynamically in Hybrid mode (both Openflow and Normal switch functioning coexist) where the CAM space could get suddenly filled up by Normal switch functioning and Openflow gets devoid of available space. **) Some deployments could have separate independent layers for HW rule insertions and application layer to interact with OVS. HW layer could face any dynamic issue during rule handling which application could not have predicted/captured in rule-construction phase. Rule-insert errors for bundles are not handled in this pull-request. Will be handled in upcoming pull request. Signed-off-by: Aravind Prasad S --- ofproto/ofproto-dpif.c | 4 ++- ofproto/ofproto-provider.h | 6 ++-- ofproto/ofproto.c | 76 +- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c index ad1e8af..d1678ed 100644 --- a/ofproto/ofproto-dpif.c +++ b/ofproto/ofproto-dpif.c @@ -4443,7 +4443,7 @@ rule_construct(struct rule *rule_) return 0; } -static void +static enum ofperr rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) OVS_REQUIRES(ofproto_mutex) { @@ -4473,6 +4473,8 @@ rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) ovs_mutex_unlock(>stats_mutex); ovs_mutex_unlock(_rule->stats_mutex); } + +return 0; } static void diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h index 2b77b89..3f3d110 100644 --- a/ofproto/ofproto-provider.h +++ b/ofproto/ofproto-provider.h @@ -1297,8 +1297,8 @@ struct ofproto_class { struct rule *(*rule_alloc)(void); enum ofperr (*rule_construct)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; -void (*rule_insert)(struct rule *rule, struct rule *old_rule, -bool forward_counts) +enum ofperr (*rule_insert)(struct rule *rule, struct rule *old_rule, + bool forward_counts) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_delete)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_destruct)(struct rule *rule); @@ -1952,7 +1952,7 @@ enum ofperr ofproto_flow_mod_learn_start(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); void ofproto_flow_mod_learn_revert(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); -void ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, +enum ofperr ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, struct ofproto *orig_ofproto) OVS_REQUIRES(ofproto_mutex); void ofproto_add_flow(struct ofproto *, const struct match *, int priority, diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c index f946e27..81b2466 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -245,10 +245,12 @@ static void replace_rule_revert(struct ofproto *, struct rule *old_rule, struct rule *new_rule) OVS_REQUIRES(ofproto_mutex); -static void replace_rule_finish(struct ofproto *, struct ofproto_flow_mod *, -const struct openflow_mod_requester *, -struct rule *old_rule, struct rule *new_rule, -struct ovs_list *dead_cookies) +static enum ofperr replace_rule_finish(struct ofproto *, + struct ofproto_flow_mod *, + const struct openflow_mod_requester *, + struct rule *old_rule, + struct rule *new_rule, + struct ovs_list *dead_cookies) OVS_REQUIRES(ofproto_mutex); static void delete_flows__(struct rule_collection *, enum ofp_flow_removed_reason, @@ -270,7 +272,7 @@ static enum ofperr ofproto_flow_mod_start(struct ofproto *, static void ofproto_flow_mod_revert(struct ofproto *, struct ofproto_flow_mod *) OVS_REQUIRES(ofproto_mutex); -static void ofproto_flow_mod_finish(struct ofproto *, +static enum ofperr ofproto_flow_mod_finish(struct ofproto *, struct ofproto_flow_mod *, const struct openflow_mod_requester *) OVS_REQUIRES(ofproto_mutex); @@ -4855,7 +4857,7 @@ add_flow_revert(struct ofproto *ofproto, struct ofproto_flow_mod *ofm) } /* To be called after version bump. */ -static void +static enum ofperr add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, const struct
Re: [ovs-dev] OVS frozen for release
On Thu, Jul 5, 2018 at 11:46 PM Han Zhou wrote: > On Mon, Jul 2, 2018 at 9:48 AM, Ben Pfaff wrote: > > > > According to our release process, we should fork branch-2.10 from master > > July 1 (yesterday), then release on August 15. I'm going to propose > > that we modify this in the same way that has been successful in the > > past, by calling for an approximately 2-week "soft freeze". During the > > freeze period, we commit only to master only bug fixes and patches that > > have been previously discussed in public before the freeze period. > > > > In this cycle, I'm proposing the following: > > > > - Now: Soft freeze begins. > > > > - July 20: Fork branch-2.10. > > > > (This is slightly late but I'm out July 13-18 and usually I'm involved > > in branching.) > > > > - August 13: Release OVS 2.10. > > > > Thanks, > > > > Ben. > > Hi, > > Here are some patches related to bug fixes I have in mind that should be in > 2.10. > > Fixing port-group: > https://patchwork.ozlabs.org/patch/931913/ > > and the follow up patch of above one: > https://patchwork.ozlabs.org/patch/934484/ > > For OVN pacemaker: > https://patchwork.ozlabs.org/patch/931228/ > https://patchwork.ozlabs.org/patch/931665/ > > > Folks may add more to this list. > Can this series - Partial cluster support in Python IDL client ( https://patchwork.ozlabs.org/project/openvswitch/list/?series=54336) be considered for OVS 2.10 ? This would be helpful for OpenStack networking-ovn, to connect to the cluster dbs. Thanks Numan > Thanks, > Han > ___ > dev mailing list > d...@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev > ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] Pyament Advice
FYI as instructed by our subsidiary company Payment swift is attached for your confirmation, kindly inform us with a return mail when you receive payment. Martin D. Dougherty Finance Manager HARDINGE INC., Add: One Hardinge Drive Elmira, NY 14902-1507 United State. Tel/Fax: +1 607-734-8819 Web: http://www.hardingeus.com ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] PRIVATE...
I have a business Proposal that will be of benefit to the both of us.Kindly contact me on michealwuu...@gmail.com should this be of interest to you. ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Re: [ovs-dev] OVS DPDK: dpdk_merge pull request for branch-2.9
0-day Robot writes: > Bleep bloop. Greetings Ian Stokes, I am a robot and I have tried out your > patch. > Thanks for your contribution. > > I encountered some error that I wasn't expecting. See the details below. > > > checkpatch: > ERROR: Too many signoffs; are you missing Co-authored-by lines? > Lines checked: 92, Warnings: 0, Errors: 1 Greetings Robot, I'm going to squelch you from working on pull requests and RFCs tomorrow. Should make get you to noopsville faster :) -Aaron ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Re: [ovs-dev] [PATCH] ofproto: Return error codes for Rule insertions
Hi Arvind, Aravind Prasad writes: > Currently, rule_insert() API doesnot have return value. There are some > possible > scenarios where rule insertions can fail at run-time even though the static > checks during rule_construct() had passed previously. > > Some possible scenarios for failure of rule insertions: > **) Rule insertions can fail dynamically in Hybrid mode (both Openflow and > Normal switch functioning coexist) where the CAM space could get suddenly > filled up by Normal switch functioning and Openflow gets devoid of > available space. > **) Some deployments could have separate independent layers for HW rule > insertions and application layer to interact with OVS. HW layer > could face any dynamic issue during rule handling which application could > not have predicted/captured in rule-construction phase. > > Rule-insert errors for bundles are not handled in this pull-request. > Will be handled in upcoming pull request. > > Signed-off-by: Aravind Prasad S > --- Thanks for working on OVS. As noted, the patch has some submission errors. Please try submitting again with 'git send-email' to eliminate any potential mail client munging. -Aaron ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] Q513
Dear Sir good greeting We are Dhiaa Alameen Company , working in the oil field in Iraq We would like to obtain a quote for the OVS Designer Licenses specified in the attached file and manufactured by you. All items are required to be quoted with Specifications in Technical part (Data Sheets / Product Detail Drawings / Product Manuals / Catalogs / Letter from Manufacturer on compliance / original confirmation). Please take care of our request and ask as soon as possible Thank you so much Mustafa Ammar Iraq - Basra -- - MUSTAFA AMMAR DHIAA ALAMEEN T COMPANY EMAIL: i...@dhialameen.com WEB : WWW.DHIALAMEEN.COM [1] [2]THIS E-MAIL MESSAGE AND ANY DOCUMENTS ATTACHED TO IT ARE CONFIDENTIAL AND MAY CONTAIN INFORMATION THAT IS PROTECTED FROM DISCLOSURE BY VARIOUS FEDERAL AND STATE LAWS, INCLUDING THE HIPAA PRIVACY RULE (45 C.F.R., PART 164). THIS INFORMATION IS INTENDED TO BE USED SOLELY BY THE ENTITY OR INDIVIDUAL TO WHOM THIS MESSAGE IS ADDRESSED. IF YOU ARE NOT THE INTENDED RECIPIENT, BE ADVISED THAT ANY USE, DISSEMINATION, FORWARDING, PRINTING, OR COPYING OF THIS MESSAGE WITHOUT THE SENDER'S WRITTEN PERMISSION IS STRICTLY PROHIBITED AND MAY BE UNLAWFUL. ACCORDINGLY, IF YOU HAVE RECEIVED THIS MESSAGE IN ERROR, PLEASE NOTIFY THE SENDER IMMEDIATELY BY CALLING 009647716104974, AND THEN DELETE THIS MESSAGE. - Links: -- [1] http://www.dhialameen.com [2] http://www.dhialameen.com/___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Re: [ovs-dev] [PATCH v2] ovs python: ovs.stream.open_block() returns success even if the remote is unreachable
This patch is now part of this series - https://patchwork.ozlabs.org/project/openvswitch/list/?series=54336 Numan On Tue, Jul 3, 2018 at 12:21 AM wrote: > From: Numan Siddique > > Calling ovs.stream.open_block(ovs.stream.open("tcp:127.0.0.1:6641")) > returns > success even if there is no server listening on 6641. To check if the > connection > is established or not, Stream class makes use of > ovs.socket_util.check_connection_completion(). > This function returns zero if the select for the socket fd signals. It > doesn't > really check if the connection was established or not. > > This patch fixes this issue by adding a wrapper function - > check_connection_completion_status() > which calls sock.connect_ex() to get the status of the connection if > ovs.socket_util.check_connection_completion() returns success. > > The test cases added fails without the fix in this patch. > > Signed-off-by: Numan Siddique > --- > python/ovs/socket_util.py | 34 ++ > python/ovs/stream.py | 16 +--- > tests/automake.mk | 1 + > tests/ovsdb-idl.at| 16 > tests/test-stream.py | 32 > 5 files changed, 96 insertions(+), 3 deletions(-) > create mode 100644 tests/test-stream.py > > v1 -> v2 - Fixed the compilation issue > > diff --git a/python/ovs/socket_util.py b/python/ovs/socket_util.py > index 403104936..91f4532ea 100644 > --- a/python/ovs/socket_util.py > +++ b/python/ovs/socket_util.py > @@ -256,6 +256,40 @@ def inet_open_active(style, target, default_port, > dscp): > return get_exception_errno(e), None > > > +def check_connection_completion_status(sock, target, default_port): > +status = check_connection_completion(sock) > +if status: > +return status > + > +try: > +address = inet_parse_active(target, default_port) > +except ValueError: > +# It's not a valid tcp target. > +return status > + > +# For tcp connections, check_connection_completion function returns 0 > +# when the select for the socket fd signals. But it doesn't really > +# verify the connection was established or not. So call connect again > on > +# the socket to get the status. > +try: > +err = sock.connect_ex(address) > +except socket.error as e: > +err = get_exception_errno(e) > +if sys.platform == 'win32' and error == errno.WSAEWOULDBLOCK: > +# WSAEWOULDBLOCK would be the equivalent on Windows > +# for EINPROGRESS on Unix. > +err = err.EINPROGRESS > + > +if err == errno.EINPROGRESS or err == errno.EALREADY: > +return errno.EINPROGRESS > + > +if err == 0 or err == errno.EISCONN: > +return 0 > + > +sock.close() > +return err > + > + > def get_exception_errno(e): > """A lot of methods on Python socket objects raise socket.error, but > that > exception is documented as having two completely different forms of > diff --git a/python/ovs/stream.py b/python/ovs/stream.py > index d6c447a97..617f31966 100644 > --- a/python/ovs/stream.py > +++ b/python/ovs/stream.py > @@ -119,6 +119,7 @@ class Stream(object): >bInitialState=False) > > self.name = name > +self.suffix = name.split(":", 1)[1] > if status == errno.EAGAIN: > self.state = Stream.__S_CONNECTING > elif status == 0: > @@ -191,8 +192,16 @@ class Stream(object): > if error: > return error, None > else: > -status = ovs.socket_util.check_connection_completion(sock) > -return 0, cls(sock, name, status) > +err = ovs.socket_util.check_connection_completion_status( > +sock, suffix, 0) > +if err == errno.EAGAIN or err == errno.EINPROGRESS: > +status = errno.EAGAIN > +err = 0 > +elif err == 0: > +status = 0 > +else: > +status = err > +return err, cls(sock, name, status) > > @staticmethod > def _open(suffix, dscp): > @@ -246,7 +255,8 @@ class Stream(object): > > def __scs_connecting(self): > if self.socket is not None: > -retval = > ovs.socket_util.check_connection_completion(self.socket) > +retval = ovs.socket_util.check_connection_completion_status( > +self.socket, self.suffix, 0) > assert retval != errno.EINPROGRESS > elif sys.platform == 'win32': > if self.retry_connect: > diff --git a/tests/automake.mk b/tests/automake.mk > index 8224e5a4a..0abf29d47 100644 > --- a/tests/automake.mk > +++ b/tests/automake.mk > @@ -421,6 +421,7 @@ CHECK_PYFILES = \ > tests/test-l7.py \ > tests/test-ovsdb.py \ > tests/test-reconnect.py \ > + tests/test-stream.py \ > tests/MockXenAPI.py \ >
[ovs-dev] [PATCH v2 2/2] python jsonrpc: Allow jsonrpc_session to have more than one remote.
From: Numan Siddique Python IDL implementation doesn't have the support to connect to the cluster dbs. This patch adds this support. We are still missing the support in python idl class to connect to the cluster master. That support will be added in an upcoming patch. This patch is similar to the commit 8cf6bbb184 which added multiple remote support in the C jsonrpc implementation. Signed-off-by: Numan Siddique --- python/ovs/jsonrpc.py | 40 +--- tests/ovsdb-idl.at| 54 +++ tests/test-ovsdb.py | 13 --- 3 files changed, 96 insertions(+), 11 deletions(-) diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py index 7c24e574a..154083766 100644 --- a/python/ovs/jsonrpc.py +++ b/python/ovs/jsonrpc.py @@ -14,6 +14,7 @@ import codecs import errno import os +import random import sys import ovs.json @@ -368,12 +369,17 @@ class Connection(object): class Session(object): """A JSON-RPC session with reconnection.""" -def __init__(self, reconnect, rpc): +def __init__(self, reconnect, rpc, remotes): self.reconnect = reconnect self.rpc = rpc self.stream = None self.pstream = None self.seqno = 0 +if type(remotes) != list: +remotes = [remotes] +self.remotes = remotes +random.shuffle(self.remotes) +self.next_remote = 0 @staticmethod def open(name, probe_interval=None): @@ -393,28 +399,39 @@ class Session(object): feature. If non-zero the value will be forced to at least 1000 milliseconds. If None it will just use the default value in OVS. """ +return Session.open_multiple(name.split(','), + probe_interval=probe_interval) + +@staticmethod +def open_multiple(remotes, probe_interval=None): reconnect = ovs.reconnect.Reconnect(ovs.timeval.msec()) -reconnect.set_name(name) +session = Session(reconnect, None, remotes) +session.pick_remote() reconnect.enable(ovs.timeval.msec()) - -if ovs.stream.PassiveStream.is_valid_name(name): +reconnect.set_backoff_free_tries(len(remotes)) +if ovs.stream.PassiveStream.is_valid_name(reconnect.get_name()): reconnect.set_passive(True, ovs.timeval.msec()) -if not ovs.stream.stream_or_pstream_needs_probes(name): +if not ovs.stream.stream_or_pstream_needs_probes(reconnect.get_name()): reconnect.set_probe_interval(0) elif probe_interval is not None: reconnect.set_probe_interval(probe_interval) -return Session(reconnect, None) +return session @staticmethod def open_unreliably(jsonrpc): reconnect = ovs.reconnect.Reconnect(ovs.timeval.msec()) +session = Session(reconnect, None, [jsonrpc.name]) reconnect.set_quiet(True) -reconnect.set_name(jsonrpc.name) +session.pick_remote() reconnect.set_max_tries(0) reconnect.connected(ovs.timeval.msec()) -return Session(reconnect, jsonrpc) +return session + +def pick_remote(self): +self.reconnect.set_name(self.remotes[self.next_remote]) +self.next_remote = (self.next_remote + 1) % len(self.remotes) def close(self): if self.rpc is not None: @@ -448,6 +465,8 @@ class Session(object): self.reconnect.connecting(ovs.timeval.msec()) else: self.reconnect.connect_failed(ovs.timeval.msec(), error) +self.stream = None +self.pick_remote() elif self.pstream is None: error, self.pstream = ovs.stream.PassiveStream.open(name) if not error: @@ -490,6 +509,7 @@ class Session(object): if error != 0: self.reconnect.disconnected(ovs.timeval.msec(), error) self.__disconnect() +self.pick_remote() elif self.stream is not None: self.stream.run() error = self.stream.connect() @@ -499,6 +519,7 @@ class Session(object): self.stream = None elif error != errno.EAGAIN: self.reconnect.connect_failed(ovs.timeval.msec(), error) +self.pick_remote() self.stream.close() self.stream = None @@ -583,3 +604,6 @@ class Session(object): def force_reconnect(self): self.reconnect.force_reconnect(ovs.timeval.msec()) + +def get_num_of_remotes(self): +return len(self.remotes) diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at index d4d283db4..d5787c5d2 100644 --- a/tests/ovsdb-idl.at +++ b/tests/ovsdb-idl.at @@ -106,6 +106,32 @@ m4_define([OVSDB_CHECK_IDL_TCP_PY], OVSDB_CHECK_IDL_TCP_PYN([$1 - Python3], [$2], [$3], [$4], [$5], [$6], [$HAVE_PYTHON3], [$PYTHON3])]) +# same as OVSDB_CHECK_IDL
[ovs-dev] [PATCH v2 1/2] ovs python: ovs.stream.open_block() returns success even if the remote is unreachable
From: Numan Siddique Calling ovs.stream.open_block(ovs.stream.open("tcp:127.0.0.1:6641")) returns success even if there is no server listening on 6641. To check if the connection is established or not, Stream class makes use of ovs.socket_util.check_connection_completion(). This function returns zero if the select for the socket fd signals. It doesn't really check if the connection was established or not. This patch fixes this issue by adding a wrapper function - check_connection_completion_status() which calls sock.connect_ex() to get the status of the connection if ovs.socket_util.check_connection_completion() returns success. The test cases added fails without the fix in this patch. Signed-off-by: Numan Siddique --- python/ovs/socket_util.py | 34 ++ python/ovs/stream.py | 16 +--- tests/automake.mk | 1 + tests/ovsdb-idl.at| 16 tests/test-stream.py | 32 5 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 tests/test-stream.py diff --git a/python/ovs/socket_util.py b/python/ovs/socket_util.py index 403104936..91f4532ea 100644 --- a/python/ovs/socket_util.py +++ b/python/ovs/socket_util.py @@ -256,6 +256,40 @@ def inet_open_active(style, target, default_port, dscp): return get_exception_errno(e), None +def check_connection_completion_status(sock, target, default_port): +status = check_connection_completion(sock) +if status: +return status + +try: +address = inet_parse_active(target, default_port) +except ValueError: +# It's not a valid tcp target. +return status + +# For tcp connections, check_connection_completion function returns 0 +# when the select for the socket fd signals. But it doesn't really +# verify the connection was established or not. So call connect again on +# the socket to get the status. +try: +err = sock.connect_ex(address) +except socket.error as e: +err = get_exception_errno(e) +if sys.platform == 'win32' and error == errno.WSAEWOULDBLOCK: +# WSAEWOULDBLOCK would be the equivalent on Windows +# for EINPROGRESS on Unix. +err = err.EINPROGRESS + +if err == errno.EINPROGRESS or err == errno.EALREADY: +return errno.EINPROGRESS + +if err == 0 or err == errno.EISCONN: +return 0 + +sock.close() +return err + + def get_exception_errno(e): """A lot of methods on Python socket objects raise socket.error, but that exception is documented as having two completely different forms of diff --git a/python/ovs/stream.py b/python/ovs/stream.py index 5996497a5..7d5227469 100644 --- a/python/ovs/stream.py +++ b/python/ovs/stream.py @@ -119,6 +119,7 @@ class Stream(object): bInitialState=False) self.name = name +self.suffix = name.split(":", 1)[1] if status == errno.EAGAIN: self.state = Stream.__S_CONNECTING elif status == 0: @@ -191,8 +192,16 @@ class Stream(object): if error: return error, None else: -status = ovs.socket_util.check_connection_completion(sock) -return 0, cls(sock, name, status) +err = ovs.socket_util.check_connection_completion_status( +sock, suffix, 0) +if err == errno.EAGAIN or err == errno.EINPROGRESS: +status = errno.EAGAIN +err = 0 +elif err == 0: +status = 0 +else: +status = err +return err, cls(sock, name, status) @staticmethod def _open(suffix, dscp): @@ -246,7 +255,8 @@ class Stream(object): def __scs_connecting(self): if self.socket is not None: -retval = ovs.socket_util.check_connection_completion(self.socket) +retval = ovs.socket_util.check_connection_completion_status( +self.socket, self.suffix, 0) assert retval != errno.EINPROGRESS elif sys.platform == 'win32': if self.retry_connect: diff --git a/tests/automake.mk b/tests/automake.mk index 8224e5a4a..0abf29d47 100644 --- a/tests/automake.mk +++ b/tests/automake.mk @@ -421,6 +421,7 @@ CHECK_PYFILES = \ tests/test-l7.py \ tests/test-ovsdb.py \ tests/test-reconnect.py \ + tests/test-stream.py \ tests/MockXenAPI.py \ tests/test-unix-socket.py \ tests/test-unixctl.py \ diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at index 58935faf3..d4d283db4 100644 --- a/tests/ovsdb-idl.at +++ b/tests/ovsdb-idl.at @@ -1720,3 +1720,19 @@ OVSDB_CHECK_IDL_COMPOUND_INDEX_WITH_REF([set, simple3 idl-compound-index-with-re 007: check simple4: empty 008: End test ]]) + +m4_define([CHECK_STREAM_OPEN_BLOCK_PY], + [AT_SETUP([$1]) + AT_SKIP_IF([test $2 = no]) +
[ovs-dev] [PATCH v2 0/2] Partial cluster support in Python IDL client
From: Numan Siddique Python IDL library is lacking the functionality to connect to the clustered db servers by providing multiple remotes (like - "tcp:10.0.0.1:6641, tcp:10.0.0.2:6641, tcp:10.0.0.3:6641") in the connection string. This patch adds this functionality to the python idl library. It still lacks the feature to connect to the master of the cluster. To add this - python idl client should monitor and read the '_Server' schema - connect to the master of the cluster. I will submit the patch once that is ready. But for now I think this is good enough for the clients to connect to the cluster dbs. v1 -> v2 Deleted the debug code which I forgot to cleanup when sending v1. Numan Siddique (2): ovs python: ovs.stream.open_block() returns success even if the remote is unreachable python jsonrpc: Allow jsonrpc_session to have more than one remote. python/ovs/jsonrpc.py | 40 +- python/ovs/socket_util.py | 34 +++ python/ovs/stream.py | 16 +++-- tests/automake.mk | 1 + tests/ovsdb-idl.at| 70 +++ tests/test-ovsdb.py | 13 ++-- tests/test-stream.py | 32 ++ 7 files changed, 192 insertions(+), 14 deletions(-) create mode 100644 tests/test-stream.py -- 2.17.1 ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] [PATCH 2/2] python jsonrpc: Allow jsonrpc_session to have more than one remote.
From: Numan Siddique Python IDL implementation doesn't have the support to connect to the cluster dbs. This patch adds this support. We are still missing the support in python idl class to connect to the cluster master. That support will be added in an upcoming patch. This patch is similar to the commit 8cf6bbb184 which added multiple remote support in the C jsonrpc implementation. Signed-off-by: Numan Siddique --- python/ovs/jsonrpc.py | 40 +++-- python/ovs/stream.py | 2 ++ tests/ovsdb-idl.at| 59 +++ tests/test-ovsdb.py | 22 +--- 4 files changed, 111 insertions(+), 12 deletions(-) diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py index 7c24e574a..154083766 100644 --- a/python/ovs/jsonrpc.py +++ b/python/ovs/jsonrpc.py @@ -14,6 +14,7 @@ import codecs import errno import os +import random import sys import ovs.json @@ -368,12 +369,17 @@ class Connection(object): class Session(object): """A JSON-RPC session with reconnection.""" -def __init__(self, reconnect, rpc): +def __init__(self, reconnect, rpc, remotes): self.reconnect = reconnect self.rpc = rpc self.stream = None self.pstream = None self.seqno = 0 +if type(remotes) != list: +remotes = [remotes] +self.remotes = remotes +random.shuffle(self.remotes) +self.next_remote = 0 @staticmethod def open(name, probe_interval=None): @@ -393,28 +399,39 @@ class Session(object): feature. If non-zero the value will be forced to at least 1000 milliseconds. If None it will just use the default value in OVS. """ +return Session.open_multiple(name.split(','), + probe_interval=probe_interval) + +@staticmethod +def open_multiple(remotes, probe_interval=None): reconnect = ovs.reconnect.Reconnect(ovs.timeval.msec()) -reconnect.set_name(name) +session = Session(reconnect, None, remotes) +session.pick_remote() reconnect.enable(ovs.timeval.msec()) - -if ovs.stream.PassiveStream.is_valid_name(name): +reconnect.set_backoff_free_tries(len(remotes)) +if ovs.stream.PassiveStream.is_valid_name(reconnect.get_name()): reconnect.set_passive(True, ovs.timeval.msec()) -if not ovs.stream.stream_or_pstream_needs_probes(name): +if not ovs.stream.stream_or_pstream_needs_probes(reconnect.get_name()): reconnect.set_probe_interval(0) elif probe_interval is not None: reconnect.set_probe_interval(probe_interval) -return Session(reconnect, None) +return session @staticmethod def open_unreliably(jsonrpc): reconnect = ovs.reconnect.Reconnect(ovs.timeval.msec()) +session = Session(reconnect, None, [jsonrpc.name]) reconnect.set_quiet(True) -reconnect.set_name(jsonrpc.name) +session.pick_remote() reconnect.set_max_tries(0) reconnect.connected(ovs.timeval.msec()) -return Session(reconnect, jsonrpc) +return session + +def pick_remote(self): +self.reconnect.set_name(self.remotes[self.next_remote]) +self.next_remote = (self.next_remote + 1) % len(self.remotes) def close(self): if self.rpc is not None: @@ -448,6 +465,8 @@ class Session(object): self.reconnect.connecting(ovs.timeval.msec()) else: self.reconnect.connect_failed(ovs.timeval.msec(), error) +self.stream = None +self.pick_remote() elif self.pstream is None: error, self.pstream = ovs.stream.PassiveStream.open(name) if not error: @@ -490,6 +509,7 @@ class Session(object): if error != 0: self.reconnect.disconnected(ovs.timeval.msec(), error) self.__disconnect() +self.pick_remote() elif self.stream is not None: self.stream.run() error = self.stream.connect() @@ -499,6 +519,7 @@ class Session(object): self.stream = None elif error != errno.EAGAIN: self.reconnect.connect_failed(ovs.timeval.msec(), error) +self.pick_remote() self.stream.close() self.stream = None @@ -583,3 +604,6 @@ class Session(object): def force_reconnect(self): self.reconnect.force_reconnect(ovs.timeval.msec()) + +def get_num_of_remotes(self): +return len(self.remotes) diff --git a/python/ovs/stream.py b/python/ovs/stream.py index 7d5227469..94c3ffd1b 100644 --- a/python/ovs/stream.py +++ b/python/ovs/stream.py @@ -22,6 +22,8 @@ import ovs.socket_util import ovs.vlog import six +import datetime + try: from OpenSSL import SSL diff --git a/tests/ovsdb-idl.at
[ovs-dev] [PATCH 1/2] ovs python: ovs.stream.open_block() returns success even if the remote is unreachable
From: Numan Siddique Calling ovs.stream.open_block(ovs.stream.open("tcp:127.0.0.1:6641")) returns success even if there is no server listening on 6641. To check if the connection is established or not, Stream class makes use of ovs.socket_util.check_connection_completion(). This function returns zero if the select for the socket fd signals. It doesn't really check if the connection was established or not. This patch fixes this issue by adding a wrapper function - check_connection_completion_status() which calls sock.connect_ex() to get the status of the connection if ovs.socket_util.check_connection_completion() returns success. The test cases added fails without the fix in this patch. Signed-off-by: Numan Siddique --- python/ovs/socket_util.py | 34 ++ python/ovs/stream.py | 16 +--- tests/automake.mk | 1 + tests/ovsdb-idl.at| 16 tests/test-stream.py | 32 5 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 tests/test-stream.py diff --git a/python/ovs/socket_util.py b/python/ovs/socket_util.py index 403104936..91f4532ea 100644 --- a/python/ovs/socket_util.py +++ b/python/ovs/socket_util.py @@ -256,6 +256,40 @@ def inet_open_active(style, target, default_port, dscp): return get_exception_errno(e), None +def check_connection_completion_status(sock, target, default_port): +status = check_connection_completion(sock) +if status: +return status + +try: +address = inet_parse_active(target, default_port) +except ValueError: +# It's not a valid tcp target. +return status + +# For tcp connections, check_connection_completion function returns 0 +# when the select for the socket fd signals. But it doesn't really +# verify the connection was established or not. So call connect again on +# the socket to get the status. +try: +err = sock.connect_ex(address) +except socket.error as e: +err = get_exception_errno(e) +if sys.platform == 'win32' and error == errno.WSAEWOULDBLOCK: +# WSAEWOULDBLOCK would be the equivalent on Windows +# for EINPROGRESS on Unix. +err = err.EINPROGRESS + +if err == errno.EINPROGRESS or err == errno.EALREADY: +return errno.EINPROGRESS + +if err == 0 or err == errno.EISCONN: +return 0 + +sock.close() +return err + + def get_exception_errno(e): """A lot of methods on Python socket objects raise socket.error, but that exception is documented as having two completely different forms of diff --git a/python/ovs/stream.py b/python/ovs/stream.py index 5996497a5..7d5227469 100644 --- a/python/ovs/stream.py +++ b/python/ovs/stream.py @@ -119,6 +119,7 @@ class Stream(object): bInitialState=False) self.name = name +self.suffix = name.split(":", 1)[1] if status == errno.EAGAIN: self.state = Stream.__S_CONNECTING elif status == 0: @@ -191,8 +192,16 @@ class Stream(object): if error: return error, None else: -status = ovs.socket_util.check_connection_completion(sock) -return 0, cls(sock, name, status) +err = ovs.socket_util.check_connection_completion_status( +sock, suffix, 0) +if err == errno.EAGAIN or err == errno.EINPROGRESS: +status = errno.EAGAIN +err = 0 +elif err == 0: +status = 0 +else: +status = err +return err, cls(sock, name, status) @staticmethod def _open(suffix, dscp): @@ -246,7 +255,8 @@ class Stream(object): def __scs_connecting(self): if self.socket is not None: -retval = ovs.socket_util.check_connection_completion(self.socket) +retval = ovs.socket_util.check_connection_completion_status( +self.socket, self.suffix, 0) assert retval != errno.EINPROGRESS elif sys.platform == 'win32': if self.retry_connect: diff --git a/tests/automake.mk b/tests/automake.mk index 8224e5a4a..0abf29d47 100644 --- a/tests/automake.mk +++ b/tests/automake.mk @@ -421,6 +421,7 @@ CHECK_PYFILES = \ tests/test-l7.py \ tests/test-ovsdb.py \ tests/test-reconnect.py \ + tests/test-stream.py \ tests/MockXenAPI.py \ tests/test-unix-socket.py \ tests/test-unixctl.py \ diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at index 58935faf3..d4d283db4 100644 --- a/tests/ovsdb-idl.at +++ b/tests/ovsdb-idl.at @@ -1720,3 +1720,19 @@ OVSDB_CHECK_IDL_COMPOUND_INDEX_WITH_REF([set, simple3 idl-compound-index-with-re 007: check simple4: empty 008: End test ]]) + +m4_define([CHECK_STREAM_OPEN_BLOCK_PY], + [AT_SETUP([$1]) + AT_SKIP_IF([test $2 = no]) +
[ovs-dev] [PATCH 0/2] Partial cluster support in Python IDL client
From: Numan Siddique Python IDL library is lacking the functionality to connect to the clustered db servers by providing multiple remotes (like - "tcp:10.0.0.1:6641, tcp:10.0.0.2:6641, tcp:10.0.0.3:6641") in the connection string. This patch adds this functionality to the python idl library. It still lacks the feature to connect to the master of the cluster. To add this - python idl client should monitor and read the '_Server' schema - connect to the master of the cluster. I will submit the patch once that is ready. But for now I think this is good enough for the clients to connect to the cluster dbs. Numan Siddique (2): ovs python: ovs.stream.open_block() returns success even if the remote is unreachable python jsonrpc: Allow jsonrpc_session to have more than one remote. python/ovs/jsonrpc.py | 40 - python/ovs/socket_util.py | 34 ++ python/ovs/stream.py | 18 -- tests/automake.mk | 1 + tests/ovsdb-idl.at| 75 +++ tests/test-ovsdb.py | 22 +--- tests/test-stream.py | 32 + 7 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 tests/test-stream.py -- 2.17.1 ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] [PATCH v2 4/4] netdev: Add a configuration option to enable dynamic rebalancing of flows
This is the fourth patch in the patch-set to support dynamic rebalancing of offloaded flows. A new OVS configuration parameter "offload-rebalance", is added to ovsdb. The default value of this is "disable". To enable this feature, set the value of this parameter to "pps-rate", which provides packets-per-second rate based policy to dynamically offload and un-offload flows. Note: This option can be enabled only when 'hw-offload' policy is enabled. It also requires 'tc-policy' to be set to 'skip_sw'; otherwise, flow offload errors (specifically ENOSPC error this feature depends on) reported by an offloaded device are supressed by TC-Flower kernel module. Signed-off-by: Sriharsha Basavapatna Co-authored-by: Venkat Duvvuru Signed-off-by: Venkat Duvvuru Reviewed-by: Sathya Perla --- lib/dpif-netlink.c| 3 ++- lib/netdev.c | 43 +++ lib/netdev.h | 2 ++ ofproto/ofproto-dpif-upcall.c | 7 -- vswitchd/vswitch.xml | 22 ++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c index 3c4b9b69f..dfe21971d 100644 --- a/lib/dpif-netlink.c +++ b/lib/dpif-netlink.c @@ -2177,7 +2177,8 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put) VLOG_DBG("added flow"); } else if (err != EEXIST) { -if (outdev && dev && (err == ENOSPC)) { +if (netdev_is_offload_rebalance_policy_enabled() && outdev && +dev && (err == ENOSPC)) { tunnel_netdev = flow_get_tunnel_netdev(); if (tunnel_netdev) { oor_netdev = tunnel_netdev; diff --git a/lib/netdev.c b/lib/netdev.c index b17f0563f..9992b846c 100644 --- a/lib/netdev.c +++ b/lib/netdev.c @@ -2474,6 +2474,43 @@ netdev_free_custom_stats_counters(struct netdev_custom_stats *custom_stats) #ifdef __linux__ +enum offload_rebalance_policy { +OFFLOAD_REBALANCE_POLICY_DISABLE, +OFFLOAD_REBALANCE_POLICY_PPS_RATE +}; + +static enum +offload_rebalance_policy netdev_offload_rebalance_policy = +OFFLOAD_REBALANCE_POLICY_DISABLE; + +static void +netdev_offload_rebalance_set_policy(const char *policy) +{ +if (!policy) { +return; +} + +if (!strcmp(policy, "pps-rate")) { +netdev_offload_rebalance_policy = OFFLOAD_REBALANCE_POLICY_PPS_RATE; +} else if (!strcmp(policy, "disable")) { +netdev_offload_rebalance_policy = OFFLOAD_REBALANCE_POLICY_DISABLE; +} else { +VLOG_WARN("netdev: Invalid policy '%s'", policy); +return; +} + +VLOG_INFO("netdev: Using policy '%s'", policy); +} + +bool +netdev_is_offload_rebalance_policy_enabled(void) +{ +if (netdev_offload_rebalance_policy == OFFLOAD_REBALANCE_POLICY_PPS_RATE) { +return true; +} +return false; +} + static void netdev_ports_flow_init(void) { @@ -2494,12 +2531,18 @@ netdev_set_flow_api_enabled(const struct smap *ovs_other_config) if (ovsthread_once_start()) { netdev_flow_api_enabled = true; +const char *offl_rebal_policy = NULL; VLOG_INFO("netdev: Flow API Enabled"); tc_set_policy(smap_get_def(ovs_other_config, "tc-policy", TC_POLICY_DEFAULT)); +offl_rebal_policy = smap_get_def(ovs_other_config, + "offload-rebalance", + OFFLOAD_REBALANCE_POLICY_DEFAULT); +netdev_offload_rebalance_set_policy(offl_rebal_policy); + netdev_ports_flow_init(); ovsthread_once_done(); diff --git a/lib/netdev.h b/lib/netdev.h index c941f1e1e..0ac2774c9 100644 --- a/lib/netdev.h +++ b/lib/netdev.h @@ -223,6 +223,8 @@ int netdev_init_flow_api(struct netdev *); uint32_t netdev_get_block_id(struct netdev *); bool netdev_is_flow_api_enabled(void); void netdev_set_flow_api_enabled(const struct smap *ovs_other_config); +bool netdev_is_offload_rebalance_policy_enabled(void); +#define OFFLOAD_REBALANCE_POLICY_DEFAULT "disable" struct dpif_port; int netdev_ports_insert(struct netdev *, const struct dpif_class *, diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c index 28596baea..49922089e 100644 --- a/ofproto/ofproto-dpif-upcall.c +++ b/ofproto/ofproto-dpif-upcall.c @@ -967,7 +967,9 @@ udpif_revalidator(void *arg) dpif_flow_dump_destroy(udpif->dump); seq_change(udpif->dump_seq); -udpif_run_flow_rebalance(udpif); +if (netdev_is_offload_rebalance_policy_enabled()) { +udpif_run_flow_rebalance(udpif); +} duration = MAX(time_msec() - start_time, 1); udpif->dump_duration = duration; @@ -2747,7 +2749,8 @@ revalidate(struct revalidator *revalidator) } ukey->dump_seq = dump_seq; -if (result !=
[ovs-dev] [PATCH v2 2/4] revalidator: Gather packets-per-second rate of flows
This is the second patch in the patch-set to support dynamic rebalancing of offloaded flows. The packets-per-second (pps) rate for each flow is computed in the context of revalidator threads when the flow stats are retrieved. The pps-rate is computed only after a flow is revalidated and is not scheduled for deletion. The parameters used to compute pps and the pps itself are saved in udpif_key since they need to be persisted across iterations of rebalancing. Signed-off-by: Sriharsha Basavapatna Co-authored-by: Venkat Duvvuru Signed-off-by: Venkat Duvvuru Reviewed-by: Sathya Perla --- lib/dpif-provider.h | 1 + ofproto/ofproto-dpif-upcall.c | 158 ++ 2 files changed, 159 insertions(+) diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h index 62b3598ac..cc6571b28 100644 --- a/lib/dpif-provider.h +++ b/lib/dpif-provider.h @@ -39,6 +39,7 @@ struct dpif { char *full_name; uint8_t netflow_engine_type; uint8_t netflow_engine_id; +long long int current_ms; }; void dpif_init(struct dpif *, const struct dpif_class *, const char *name, diff --git a/ofproto/ofproto-dpif-upcall.c b/ofproto/ofproto-dpif-upcall.c index 85f579251..727d9f508 100644 --- a/ofproto/ofproto-dpif-upcall.c +++ b/ofproto/ofproto-dpif-upcall.c @@ -42,6 +42,8 @@ #include "tunnel.h" #include "unixctl.h" #include "openvswitch/vlog.h" +#include "lib/dpif-provider.h" +#include "lib/netdev-provider.h" #define MAX_QUEUE_LENGTH 512 #define UPCALL_MAX_BATCH 64 @@ -304,6 +306,16 @@ struct udpif_key { uint32_t key_recirc_id; /* Non-zero if reference is held by the ukey. */ struct recirc_refs recircs; /* Action recirc IDs with references held. */ + +#define OFFL_REBAL_INTVL_MSEC 3000/* dynamic offload rebalance freq */ +struct netdev *in_netdev; /* in_odp_port's netdev */ +struct netdev *out_netdev; /* out_odp_port's netdev */ +struct netdev *tunnel_netdev; /* tunnel netdev */ +bool offloaded;/* True if flow is offloaded */ +float flow_pps_rate; /* Packets-Per-Second rate */ +long long int flow_time; /* last pps update time */ +uint64_t flow_packets; /* #pkts seen in interval */ +uint64_t flow_backlog_packets; /* prev-mode #pkts (offl or kernel) */ }; /* Datapath operation with optional ukey attached. */ @@ -1667,6 +1679,11 @@ ukey_create__(const struct nlattr *key, size_t key_len, ukey->stats.used = used; ukey->xcache = NULL; +ukey->offloaded = false; +ukey->flow_time = 0; +ukey->in_netdev = ukey->out_netdev = ukey->tunnel_netdev = NULL; +ukey->flow_packets = ukey->flow_backlog_packets = 0; + ukey->key_recirc_id = key_recirc_id; recirc_refs_init(>recircs); if (xout) { @@ -2442,6 +2459,143 @@ reval_op_init(struct ukey_op *op, enum reval_result result, } } +/* + * Given a dpif_flow, get its input and output ports (netdevs) by parsing + * the flow keys and actions. Save them in udpif_key. + */ +static void +dpif_flow_to_netdevs(struct dpif *dpif, struct udpif_key *ukey, + struct dpif_flow *f) +{ +const struct dpif_class *dpif_class = dpif->dpif_class; +odp_port_t out_port = ODPP_NONE; +odp_port_t in_port = ODPP_NONE; +const struct nlattr *a; +const struct nlattr *k; +bool forward = false; +unsigned int left; + +ukey->in_netdev = NULL; +ukey->out_netdev = NULL; + +/* Capture the output port */ +NL_ATTR_FOR_EACH (a, left, f->actions, f->actions_len) { +enum ovs_action_attr type = nl_attr_type(a); +if (type == OVS_ACTION_ATTR_OUTPUT) { +/* Only unicast is supported */ +if (forward) { +ukey->out_netdev = NULL; +ukey->in_netdev = NULL; +return; +} + +forward = true; +out_port = nl_attr_get_odp_port(a); +ukey->out_netdev = netdev_ports_get(out_port, dpif_class); +break; +} +} + +/* Now find the input port */ +NL_ATTR_FOR_EACH (k, left, f->key, f->key_len) { +unsigned int type; +struct flow_tnl tnl; +enum odp_key_fitness res; + +type = nl_attr_type(k); + +switch (type) { +case OVS_KEY_ATTR_IN_PORT: +in_port = *(odp_port_t *)nl_attr_get(k); +ukey->in_netdev = netdev_ports_get(in_port, dpif_class); +VLOG_DBG("%s: in_netdev: %s\n", __func__, ukey->in_netdev->name); +break; +case OVS_KEY_ATTR_TUNNEL: +res = odp_tun_key_from_attr(k, ); +if (res == ODP_FIT_ERROR) { +ukey->tunnel_netdev = NULL; +break; +} +ukey->tunnel_netdev = flow_get_tunnel_netdev(); +VLOG_DBG("%s: tunnel_netdev: %s\n", __func__, + ukey->tunnel_netdev->name); +break; +default:
[ovs-dev] [PATCH v2 3/4] revalidator: Rebalance offloaded flows based on the pps rate
This is the third patch in the patch-set to support dynamic rebalancing of offloaded flows. The dynamic rebalancing functionality is implemented in this patch. The ukeys that are not scheduled for deletion are obtained and passed as input to the rebalancing routine. The rebalancing is done in the context of revalidation leader thread, after all other revalidator threads are done with gathering rebalancing data for flows. For each netdev that is in OOR state, a list of flows - both offloaded and non-offloaded (pending) - is obtained using the ukeys. For each netdev that is in OOR state, the flows are grouped and sorted into offloaded and pending flows. The offloaded flows are sorted in descending order of pps-rate, while pending flows are sorted in ascending order of pps-rate. The rebalancing is done in two phases. In the first phase, we try to offload all pending flows and if that succeeds, the OOR state on the device is cleared. If some (or none) of the pending flows could not be offloaded, then we start replacing an offloaded flow that has a lower pps-rate than a pending flow, until there are no more pending flows with a higher rate than an offloaded flow. The flows that are replaced from the device are added into kernel datapath. Signed-off-by: Sriharsha Basavapatna Co-authored-by: Venkat Duvvuru Signed-off-by: Venkat Duvvuru Reviewed-by: Sathya Perla --- lib/dpif-netdev.c | 3 +- lib/dpif-netlink.c| 14 +- lib/dpif-provider.h | 7 +- lib/dpif.c| 20 +- lib/dpif.h| 20 +- lib/netdev-provider.h | 3 +- ofproto/ofproto-dpif-upcall.c | 434 +- 7 files changed, 480 insertions(+), 21 deletions(-) diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c index 9390fff68..ad5aac62b 100644 --- a/lib/dpif-netdev.c +++ b/lib/dpif-netdev.c @@ -3052,7 +3052,8 @@ dpif_netdev_execute(struct dpif *dpif, struct dpif_execute *execute) } static void -dpif_netdev_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops) +dpif_netdev_operate(struct dpif *dpif, struct dpif_op **ops, size_t n_ops, +enum dpif_op_skip_type skip_flag OVS_UNUSED) { size_t i; diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c index 5a6d53ad5..3c4b9b69f 100644 --- a/lib/dpif-netlink.c +++ b/lib/dpif-netlink.c @@ -2276,7 +2276,8 @@ dpif_netlink_operate_chunks(struct dpif_netlink *dpif, struct dpif_op **ops, } static void -dpif_netlink_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) +dpif_netlink_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops, + enum dpif_op_skip_type skip_flag) { struct dpif_netlink *dpif = dpif_netlink_cast(dpif_); struct dpif_op *new_ops[OPERATE_MAX_OPS]; @@ -2284,7 +2285,7 @@ dpif_netlink_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) int i = 0; int err = 0; -if (netdev_is_flow_api_enabled()) { +if (skip_flag != DPIF_OP_SKIP_OFFLOAD && netdev_is_flow_api_enabled()) { while (n_ops > 0) { count = 0; @@ -2293,6 +2294,10 @@ dpif_netlink_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) err = try_send_to_netdev(dpif, op); if (err && err != EEXIST) { +if (skip_flag == DPIF_OP_SKIP_DP) { +op->error = err; +return; +} new_ops[count++] = op; } else { op->error = err; @@ -2303,8 +2308,11 @@ dpif_netlink_operate(struct dpif *dpif_, struct dpif_op **ops, size_t n_ops) dpif_netlink_operate_chunks(dpif, new_ops, count); } -} else { +} else if (skip_flag != DPIF_OP_SKIP_DP) { dpif_netlink_operate_chunks(dpif, ops, n_ops); +} else { +VLOG_ERR("%s: Invalid skip_flag: %d while flow api is disabled\n", + __func__, skip_flag); } } diff --git a/lib/dpif-provider.h b/lib/dpif-provider.h index cc6571b28..2bbfcf7f6 100644 --- a/lib/dpif-provider.h +++ b/lib/dpif-provider.h @@ -296,12 +296,13 @@ struct dpif_class { int (*flow_dump_next)(struct dpif_flow_dump_thread *thread, struct dpif_flow *flows, int max_flows); - /* Executes each of the 'n_ops' operations in 'ops' on 'dpif', in the order * in which they are specified, placing each operation's results in the * "output" members documented in comments and the 'error' member of each - * dpif_op. */ -void (*operate)(struct dpif *dpif, struct dpif_op **ops, size_t n_ops); + * dpif_op. The skip_flag argument tells the provider if 'ops' should be + * offloaded to a netdev or to the kernel datapath or to both. */ +void (*operate)(struct dpif *dpif, struct dpif_op **ops, size_t n_ops, +enum dpif_op_skip_type skip_flag); /* Enables or disables
[ovs-dev] [PATCH v2 1/4] dpif-netlink: Detect Out-Of-Resource condition on a netdev
This is the first patch in the patch-set to support dynamic rebalancing of offloaded flows. The patch detects OOR condition on a netdev port when ENOSPC error is returned by TC-Flower while adding a flow rule. A new structure is added to the netdev called "netdev_hw_info", to store OOR related information required to perform dynamic offload-rebalancing. Signed-off-by: Sriharsha Basavapatna Co-authored-by: Venkat Duvvuru Signed-off-by: Venkat Duvvuru Reviewed-by: Sathya Perla --- lib/dpif-netlink.c| 22 ++ lib/flow.c| 27 +++ lib/flow.h| 1 + lib/netdev-provider.h | 7 +++ lib/netdev.c | 2 ++ 5 files changed, 55 insertions(+), 4 deletions(-) diff --git a/lib/dpif-netlink.c b/lib/dpif-netlink.c index aa9bbd946..5a6d53ad5 100644 --- a/lib/dpif-netlink.c +++ b/lib/dpif-netlink.c @@ -2097,11 +2097,16 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put) { static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20); const struct dpif_class *dpif_class = dpif->dpif.dpif_class; +struct netdev_hw_info *hw_info; struct match match; odp_port_t in_port; +odp_port_t out_port; const struct nlattr *nla; size_t left; struct netdev *dev; +struct netdev *outdev = NULL; +struct netdev *tunnel_netdev = NULL; +struct netdev *oor_netdev = NULL; struct offload_info info; ovs_be16 dst_port = 0; int err; @@ -2131,8 +2136,6 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put) NL_ATTR_FOR_EACH(nla, left, put->actions, put->actions_len) { if (nl_attr_type(nla) == OVS_ACTION_ATTR_OUTPUT) { const struct netdev_tunnel_config *tnl_cfg; -struct netdev *outdev; -odp_port_t out_port; out_port = nl_attr_get_odp_port(nla); outdev = netdev_ports_get(out_port, dpif_class); @@ -2144,7 +2147,6 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put) if (tnl_cfg && tnl_cfg->dst_port != 0) { dst_port = tnl_cfg->dst_port; } -netdev_close(outdev); } } @@ -2175,7 +2177,18 @@ parse_flow_put(struct dpif_netlink *dpif, struct dpif_flow_put *put) VLOG_DBG("added flow"); } else if (err != EEXIST) { -VLOG_ERR_RL(, "failed to offload flow: %s", ovs_strerror(err)); +if (outdev && dev && (err == ENOSPC)) { +tunnel_netdev = flow_get_tunnel_netdev(); +if (tunnel_netdev) { +oor_netdev = tunnel_netdev; +} else { +oor_netdev = dev; +} +hw_info = _netdev->hw_info; +hw_info->oor = true; +} +VLOG_ERR_RL(, "failed to offload flow: %s: %s", ovs_strerror(err), +(oor_netdev ? oor_netdev->name : dev->name)); } out: @@ -2196,6 +2209,7 @@ out: } } +netdev_close(outdev); netdev_close(dev); return err; diff --git a/lib/flow.c b/lib/flow.c index 75ca45672..912afc99d 100644 --- a/lib/flow.c +++ b/lib/flow.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,8 @@ #include "unaligned.h" #include "util.h" #include "openvswitch/nsh.h" +#include "ovs-router.h" +#include "lib/netdev-provider.h" COVERAGE_DEFINE(flow_extract); COVERAGE_DEFINE(miniflow_malloc); @@ -3299,3 +3302,27 @@ flow_limit_vlans(int vlan_limit) flow_vlan_limit = MIN(vlan_limit, FLOW_MAX_VLAN_HEADERS); } } + +struct netdev * +flow_get_tunnel_netdev(struct flow_tnl *tunnel) +{ +struct netdev *tunnel_netdev; +char iface[IFNAMSIZ]; +struct in6_addr ip6; +struct in6_addr gw; + +if (tunnel->ip_src) { +in6_addr_set_mapped_ipv4(, tunnel->ip_src); +} else if (ipv6_addr_is_set(>ipv6_src)) { +ip6 = tunnel->ipv6_src; +} else { +return NULL; +} + +if (!ovs_router_lookup(0, , iface, NULL, )) { +return (NULL); +} + +tunnel_netdev = netdev_from_name(iface); +return tunnel_netdev; +} diff --git a/lib/flow.h b/lib/flow.h index 5b6585f11..a67abc9c9 100644 --- a/lib/flow.h +++ b/lib/flow.h @@ -73,6 +73,7 @@ void flow_extract(struct dp_packet *, struct flow *); void flow_zero_wildcards(struct flow *, const struct flow_wildcards *); void flow_unwildcard_tp_ports(const struct flow *, struct flow_wildcards *); void flow_get_metadata(const struct flow *, struct match *flow_metadata); +struct netdev *flow_get_tunnel_netdev(struct flow_tnl *tunnel); const char *ct_state_to_string(uint32_t state); uint32_t ct_state_from_string(const char *); diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h index 1a572f5b8..62e05619e 100644 --- a/lib/netdev-provider.h +++ b/lib/netdev-provider.h @@ -35,6 +35,11 @@ extern "C" { struct netdev_tnl_build_header_params; #define
[ovs-dev] [PATCH v2 0/4] Support dynamic rebalancing of offloaded flows
v1 -> v2 changes: - Fixed build errors reported by 0-day robot - Updated patch prefixes with relevant subsystem names ** With the current OVS offload design, when an offload-device fails to add a flow rule and returns an error, OVS adds the rule to the kernel datapath. The flow gets processed by the kernel datapath for the entire life of that flow. This is fine when an error is returned by the device due to lack of support for certain keys or actions. But when an error is returned due to temporary conditions such as lack of resources to add a flow rule, the flow continues to be processed by kernel even when resources become available later. That is, those flows never get offloaded again. This problem becomes more pronounced when a flow that has been initially offloaded may have a smaller packet rate than a later flow that could not be offloaded due to lack of resources. This leads to inefficient use of HW resources and wastage of host CPU cycles. This patch-set addresses this issue by providing a way to detect temporary offload resource constraints (Out-Of-Resource or OOR condition) and to selectively and dynamically offload flows with a higher packets-per-second (pps) rate. This dynamic rebalancing is done periodically on netdevs that are in OOR state until resources become available to offload all pending flows. The patch-set involves the following changes at a high level: 1. Detection of Out-Of-Resources (OOR) condition on an offload-capable netdev. 2. Gathering flow offload selection criteria for all flows on an OOR netdev; i.e, packets-per-second (pps) rate of flows for offloaded and non-offloaded (pending) flows. 3. Dynamically replacing offloaded flows with a lower pps-rate, with non-offloaded flows with a higher pps-rate, on an OOR netdev. 4. A new OpenvSwitch configuration option - "offload-rebalancing" to enable this policy. ** Sriharsha Basavapatna (4): dpif-netlink: Detect Out-Of-Resource condition on a netdev revalidator: Gather packets-per-second rate of flows revalidator: Rebalance offloaded flows based on the pps rate netdev: Add a configuration option to enable dynamic rebalancing of flows lib/dpif-netdev.c | 3 +- lib/dpif-netlink.c| 37 ++- lib/dpif-provider.h | 8 +- lib/dpif.c| 20 +- lib/dpif.h| 20 +- lib/flow.c| 27 ++ lib/flow.h| 1 + lib/netdev-provider.h | 8 + lib/netdev.c | 45 +++ lib/netdev.h | 2 + ofproto/ofproto-dpif-upcall.c | 593 +- vswitchd/vswitch.xml | 22 ++ 12 files changed, 763 insertions(+), 23 deletions(-) -- 2.18.0.rc1.1.g6f333ff ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
Re: [ovs-dev] ofproto: Return error codes for Rule insertions
Bleep bloop. Greetings Aravind Prasad, I am a robot and I have tried out your patch. Thanks for your contribution. I encountered some error that I wasn't expecting. See the details below. git-am: fatal: patch fragment without header at line 7: @@ -270,7 +272,7 @@ static enum ofperr ofproto_flow_mod_start(struct Repository lacks necessary blobs to fall back on 3-way merge. Cannot fall back to three-way merge. Patch failed at 0001 ofproto: Return error codes for Rule insertions The copy of the patch that failed is found in: /var/lib/jenkins/jobs/upstream_build_from_pw/workspace/.git/rebase-apply/patch When you have resolved this problem, run "git am --resolved". If you prefer to skip this patch, run "git am --skip" instead. To restore the original branch and stop patching, run "git am --abort". Please check this out. If you feel there has been an error, please email acon...@bytheb.org Thanks, 0-day Robot ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev
[ovs-dev] [PATCH] ofproto: Return error codes for Rule insertions
Currently, rule_insert() API doesnot have return value. There are some possible scenarios where rule insertions can fail at run-time even though the static checks during rule_construct() had passed previously. Some possible scenarios for failure of rule insertions: **) Rule insertions can fail dynamically in Hybrid mode (both Openflow and Normal switch functioning coexist) where the CAM space could get suddenly filled up by Normal switch functioning and Openflow gets devoid of available space. **) Some deployments could have separate independent layers for HW rule insertions and application layer to interact with OVS. HW layer could face any dynamic issue during rule handling which application could not have predicted/captured in rule-construction phase. Rule-insert errors for bundles are not handled in this pull-request. Will be handled in upcoming pull request. Signed-off-by: Aravind Prasad S --- diff --git a/ofproto/ofproto-dpif.c b/ofproto/ofproto-dpif.c index ad1e8af..d1678ed 100644 --- a/ofproto/ofproto-dpif.c +++ b/ofproto/ofproto-dpif.c @@ -4443,7 +4443,7 @@ rule_construct(struct rule *rule_) return 0; } -static void +static enum ofperr rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) OVS_REQUIRES(ofproto_mutex) { @@ -4473,6 +4473,8 @@ rule_insert(struct rule *rule_, struct rule *old_rule_, bool forward_counts) ovs_mutex_unlock(>stats_mutex); ovs_mutex_unlock(_rule->stats_mutex); } + +return 0; } static void diff --git a/ofproto/ofproto-provider.h b/ofproto/ofproto-provider.h index 2b77b89..3f3d110 100644 --- a/ofproto/ofproto-provider.h +++ b/ofproto/ofproto-provider.h @@ -1297,8 +1297,8 @@ struct ofproto_class { struct rule *(*rule_alloc)(void); enum ofperr (*rule_construct)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; -void (*rule_insert)(struct rule *rule, struct rule *old_rule, -bool forward_counts) +enum ofperr (*rule_insert)(struct rule *rule, struct rule *old_rule, + bool forward_counts) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_delete)(struct rule *rule) /* OVS_REQUIRES(ofproto_mutex) */; void (*rule_destruct)(struct rule *rule); @@ -1952,7 +1952,7 @@ enum ofperr ofproto_flow_mod_learn_start(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); void ofproto_flow_mod_learn_revert(struct ofproto_flow_mod *ofm) OVS_REQUIRES(ofproto_mutex); -void ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, +enum ofperr ofproto_flow_mod_learn_finish(struct ofproto_flow_mod *ofm, struct ofproto *orig_ofproto) OVS_REQUIRES(ofproto_mutex); void ofproto_add_flow(struct ofproto *, const struct match *, int priority, diff --git a/ofproto/ofproto.c b/ofproto/ofproto.c index f946e27..81b2466 100644 --- a/ofproto/ofproto.c +++ b/ofproto/ofproto.c @@ -245,10 +245,12 @@ static void replace_rule_revert(struct ofproto *, struct rule *old_rule, struct rule *new_rule) OVS_REQUIRES(ofproto_mutex); -static void replace_rule_finish(struct ofproto *, struct ofproto_flow_mod *, -const struct openflow_mod_requester *, -struct rule *old_rule, struct rule *new_rule, -struct ovs_list *dead_cookies) +static enum ofperr replace_rule_finish(struct ofproto *, + struct ofproto_flow_mod *, + const struct openflow_mod_requester *, + struct rule *old_rule, + struct rule *new_rule, + struct ovs_list *dead_cookies) OVS_REQUIRES(ofproto_mutex); static void delete_flows__(struct rule_collection *, enum ofp_flow_removed_reason, @@ -270,7 +272,7 @@ static enum ofperr ofproto_flow_mod_start(struct ofproto *, static void ofproto_flow_mod_revert(struct ofproto *, struct ofproto_flow_mod *) OVS_REQUIRES(ofproto_mutex); -static void ofproto_flow_mod_finish(struct ofproto *, +static enum ofperr ofproto_flow_mod_finish(struct ofproto *, struct ofproto_flow_mod *, const struct openflow_mod_requester *) OVS_REQUIRES(ofproto_mutex); @@ -4855,7 +4857,7 @@ add_flow_revert(struct ofproto *ofproto, struct ofproto_flow_mod *ofm) } /* To be called after version bump. */ -static void +static enum ofperr add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, const struct openflow_mod_requester *req) OVS_REQUIRES(ofproto_mutex) @@ -4864,8 +4866,14 @@ add_flow_finish(struct ofproto *ofproto, struct ofproto_flow_mod *ofm, ? rule_collection_rules(>old_rules)[0] : NULL; struct
[ovs-dev] urgent for your photos
We would like to introduce our image editing services for you. We would appreciate if you can reply with your requirements and can give us a chance to serve you. Our mainly services are: . Cut out, masking, clipping path, deep etching, transparent background . Colour correction, black and white, light and shadows etc. . Dust cleaning, spot cleaning . Beauty retouching, skin retouching, face retouching, body retouching . Fashion/Beauty Image Retouching . Product image Retouching . Real estate image Retouching . Wedding & Event Album Design. . Vector Conversion . Portrait image Retouching We deliver the work within 24-48 hours. We can give you editing test on your photos. Please reply if you have interests. Thanks, Julie ___ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev