Hi Raja,
Thanks for respone. I see that now mpls label is different, so I'am sending
fresh output
root@cn-a-0-28:~# rt --dump 1 | egrep "10.189.85.19\/32|Dest"
Destination PPL Flags Label Nexthop
Stitched MAC(Index)
10.189.85.19/32 32 LP 16 17
2:59:4f:9c:af:9e(100440)
root@cn-a-0-27:~# mpls --get 16
MPLS Input Label Map
Label NextHop
-------------------
16 12
root@cn-a-0-27:~# nh --get 12
Id:12 Type:Encap Fmly: AF_INET Flags:Valid, Policy, Rid:0
Ref_cnt:5 Vrf:1
EncapFmly:0806 Oif:3 Len:14 Data:02 59 4f 9c af 9e 00 00 5e
00 01 00 08 00
root@cn-a-0-27:~# vif --get 3
Vrouter Interface Table
Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
L2=Layer 2
D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
Offload, Mon=Interface is Monitored
Uuf=Unknown Unicast Flood
vif0/3 OS: tap594f9caf-9e
Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
Vrf:1 Flags:PL3L2D MTU:9160 Ref:6
RX packets:19 bytes:5248 errors:0
TX packets:13 bytes:798 errors:0
It seems to be correct.
For old label
root@cn-a-0-27:~# mpls --get 22
MPLS Input Label Map
Label NextHop
-------------------
22 36
root@cn-a-0-27:~# nh --get 36
Id:36 Type:Encap Fmly:AF_BRIDGE Flags:Valid, Policy, Rid:0
Ref_cnt:3 Vrf:1
EncapFmly:0806 Oif:5 Len:14 Data:02 25 46 1e 6e 47 00 00 5e
00 01 00 08 00
root@cn-a-0-27:~# vif --get 5
Vrouter Interface Table
Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
L2=Layer 2
D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
Offload, Mon=Interface is Monitored
Uuf=Unknown Unicast Flood
vif0/5 OS: tap25461e6e-47
Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
Vrf:1 Flags:PL3L2D MTU:9160 Ref:6
RX packets:16 bytes:1056 errors:0
TX packets:8 bytes:336 errors:0
Debug log
<4>Oct 20 10:19:53 cn-a-0-27 kernel: [68521.966480] SEND PACKET TO AGENT
vif_idx=2, vif_os_idx=101, vif_nh_id=0, vif_mac=, vif_name=pkt0, vif_ip=0
<4>Oct 20 10:19:53 cn-a-0-27 kernel: [68521.966861] if(fe) forward:1
<4>Oct 20 10:19:53 cn-a-0-27 kernel: [68521.966864] vr_gro_input: nh_id: 118
<4>Oct 20 10:19:53 cn-a-0-27 kernel: [68521.966871] pkt_gro_dev_rx_handler
nh_id:118
<2>Oct 20 10:19:53 cn-a-0-27 kernel: [68521.966874] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=19 vif_os_idx=22 vif_mac=
vif_name=tapbf8e6fea-5b vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.968798] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.968844] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.968857] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.982541] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.982575] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.982596] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.982615] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.995646] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68522.995647] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.009229] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.009243] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.009279] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.009301] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.009585] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.022347] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.022348] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.031691] SEND PACKET TO AGENT
vif_idx=2, vif_os_idx=101, vif_nh_id=0, vif_mac=, vif_name=pkt0, vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.031984] if(fe) forward:0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.035381] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.035382] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.048961] vr_gro_input: nh_id: 15
<4>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.049011] pkt_gro_dev_rx_handler
nh_id:15
<2>Oct 20 10:19:54 cn-a-0-27 kernel: [68523.049027] pkt_gro_dev_rx_handler:
vif nexthop info: vif_type=3 vif_idx=3 vif_os_idx=34 vif_mac=
vif_name=tap594f9caf-9e vif_ip=0
source/dp-core/vr_datapath.c
int
vr_gro_input(struct vr_packet *pkt, struct vr_nexthop *nh)
{
unsigned short *nh_id;
int handled = 1;
if (!vr_gro_process)
return !handled;
nh_id = (unsigned short *)pkt_push(pkt, sizeof(*nh_id));
if (!nh_id) {
vr_pfree(pkt, VP_DROP_PUSH);
return handled;
}
*nh_id = nh->nh_id;
printk("vr_gro_input: nh_id: %d\n", *nh_id);
handled = vr_gro_process(pkt, nh->nh_dev, (nh->nh_family == AF_BRIDGE));
return handled;
}
source/linux/vr_host_interface.c
static rx_handler_result_t
pkt_gro_dev_rx_handler(struct sk_buff **pskb)
{
unsigned short nh_id, drop_reason;
struct vr_nexthop *nh;
struct vr_interface *vif;
struct vr_interface *gro_vif;
struct vr_interface_stats *gro_vif_stats;
struct sk_buff *skb = *pskb;
struct vr_packet *pkt = NULL;
struct vrouter *router = vrouter_get(0);
struct vr_forwarding_md fmd;
#ifdef XEN_HYPERVISOR
unsigned char *data;
data = skb_mac_header(skb);
nh_id = *((unsigned short *)(data + (ETH_HLEN - sizeof(unsigned
short))));
if (!skb_push(skb, VR_ETHER_HLEN)) {
drop_reason = VP_DROP_INVALID_PACKET;
goto drop;
}
#else
nh_id = *((unsigned short *) skb_mac_header(skb));
#endif
printk("pkt_gro_dev_rx_handler nh_id:%d\n", nh_id);
nh = __vrouter_get_nexthop(router, nh_id);
Kind Regards,
Jacek
2015-10-20 9:02 GMT+02:00 Rajagopalan Sivaramakrishnan <[email protected]>:
> Hi Jacek,
> On cn-a-0-27, can you please run ³mpls ‹dump² to get the next hop
> corresponding to label 22?
> And check whether this is the nh_id that vr_gro_input() and
> pkt_gro_dev_rx_handler() see?
>
> Raja
>
> On 10/18/15, 11:45 AM, "Users on behalf of Jacek Czerniak"
> <[email protected] on behalf of [email protected]>
> wrote:
>
> >Hi Raja,
> >
> >Please find below further explanation of our problem.
> >
> >
> >Legend:
> >cn-a-0-28 - source compute node, underlay ip:
> >10.187.1.28
> >cn-a-0-27 - destination compute node, underlay ip:
> >10.187.1.27
> >
> >SourceVM - located on cn-a-0-28, ip: 10.189.85.25/24, mac:
> >02:a8:f3:9c:c1:01, tap: tapa8f39cc1-01
> >DestinationVM - located on cn-a-0-27, ip: 10.189.85.19/24, mac:
> >02:59:4f:9c:af:9e, tap: tap594f9caf-9e
> >ProblematicVM - located on cn-a-0-27, ip: 10.189.85.22/24, mac:
> >02:0c:f1:96:75:22, tap: tap0cf19675-22
> >
> >
> >Test traffic HTTP:
> >
> >[root@SourceVM~]# time curl -s -o /dev/null 'http://10.189.85.19/'
> >
> >real 0m1.010s
> >user 0m0.003s
> >sys 0m0.003s
> >
> >Respone after 0m1.010s
> >
> >
> >#Traffic dump:
> >#ProblematicVM - get first SYN
> >
> >root@cn-a-0-27:~# tcpdump -eni tap0cf19675-22 port 80
> >tcpdump: WARNING: tap0cf19675-22: no IPv4 address assigned
> >tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
> >listening on tap0cf19675-22, link-type EN10MB (Ethernet), capture size
> >65535 bytes
> >
> >18:57:19.364293 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 74: 10.189.85.25.58260 > 10.189.85.19.80: Flags [S],
> >seq 3662446385, win 14600, options [mss 1460,sackOK,TS val 269329475 ecr
> >0,nop,wscale 7], length 0
> >
> >#DestinationVM - after second SYN established connection
> >
> >root@cn-a-0-27:~# tcpdump -eni tap594f9caf-9e port 80 [
> >tcpdump: WARNING: tap594f9caf-9e: no IPv4 address assigned
> >tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
> >listening on tap594f9caf-9e, link-type EN10MB (Ethernet), capture size
> >65535 bytes
> >
> >18:57:20.363719 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 74: 10.189.85.25.58260 > 10.189.85.19.80: Flags [S],
> >seq 3662446385, win 14600,
> >options [mss 1460,sackOK,TS val 269330476 ecr 0,nop,wscale 7], length 0
> >
> >18:57:20.363944 02:59:4f:9c:af:9e > 02:a8:f3:9c:c1:01, ethertype IPv4
> >(0x0800), length 74: 10.189.85.19.80 > 10.189.85.25.58260: Flags [S.],
> >seq 1036913679, ack 366244
> >6386, win 14480, options [mss 1460,sackOK,TS val 190985773 ecr
> >269330476,nop,wscale 7], length 0
> >18:57:20.364312 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 66: 10.189.85.25.58260 > 10.189.85.19.80: Flags [.],
> >ack 1, win 115, options [no
> >p,nop,TS val 269330476 ecr 190985773], length 0
> >18:57:20.364325 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 142: 10.189.85.25.58260 > 10.189.85.19.80: Flags [P.],
> >seq 1:77, ack 1, win 115,
> > options [nop,nop,TS val 269330476 ecr 190985773], length 76
> >18:57:20.364446 02:59:4f:9c:af:9e > 02:a8:f3:9c:c1:01, ethertype IPv4
> >(0x0800), length 66: 10.189.85.19.80 > 10.189.85.25.58260: Flags [.],
> >ack 77, win 114, options [n
> >op,nop,TS val 190985774 ecr 269330476], length 0
> >18:57:20.364778 02:59:4f:9c:af:9e > 02:a8:f3:9c:c1:01, ethertype IPv4
> >(0x0800), length 2962: 10.189.85.19.80 > 10.189.85.25.58260: Flags [.],
> >seq 1:2897, ack 77, win 1
> >14, options [nop,nop,TS val 190985774 ecr 269330476], length 2896
> >18:57:20.364850 02:59:4f:9c:af:9e > 02:a8:f3:9c:c1:01, ethertype IPv4
> >(0x0800), length 1108: 10.189.85.19.80 > 10.189.85.25.58260: Flags [P.],
> >seq 2897:3939, ack 77, w
> >in 114, options [nop,nop,TS val 190985774 ecr 269330476], length 1042
> >18:57:20.365135 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 66: 10.189.85.25.58260 > 10.189.85.19.80: Flags [.],
> >ack 2897, win 160, options
> >
> >...
> >
> >
> >#Test traffic ICMP
> >
> >[root@SourceVM ~]# ping 10.189.85.19 -c 2 -W 1
> >PING 10.189.85.19 (10.189.85.19) 56(84) bytes of data.
> >64 bytes from 10.189.85.19: icmp_seq=2 ttl=64 time=0.676 ms
> >
> >--- 10.189.85.19 ping statistics ---
> >2 packets transmitted, 1 received, 50% packet loss, time 999ms
> >rtt min/avg/max/mdev = 0.676/0.676/0.676/0.000 ms
> >
> >
> >#ProblematicVM
> >root@cn-a-0-27:~# tcpdump -eni tap0cf19675-22 icmp
> >tcpdump: WARNING: tap0cf19675-22: no IPv4 address assigned
> >tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
> >listening on tap0cf19675-22, link-type EN10MB (Ethernet), capture size
> >65535 bytes
> >20:15:48.819471 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 98: 10.189.85.25 > 10.189.85.19: ICMP echo request, id
> >9075, seq 1, length 64
> >
> >#DestinationVM
> >root@cn-a-0-27:~# tcpdump -eni tap594f9caf-9e icmp
> >tcpdump: WARNING: tap594f9caf-9e: no IPv4 address assigned
> >tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
> >listening on tap594f9caf-9e, link-type EN10MB (Ethernet), capture size
> >65535 bytes
> >20:15:49.816581 02:a8:f3:9c:c1:01 > 02:59:4f:9c:af:9e, ethertype IPv4
> >(0x0800), length 98: 10.189.85.25 > 10.189.85.19: ICMP echo request, id
> >9075, seq 2, length 64
> >20:15:49.816890 02:59:4f:9c:af:9e > 02:a8:f3:9c:c1:01, ethertype IPv4
> >(0x0800), length 98: 10.189.85.19 > 10.189.85.25: ICMP echo reply, id
> >9075, seq 2, length 64
> >
> >
> >
> >
> >Our debug steps:
> >
> >## Check path from source compute node
> >## Get source vif, rt and nh for DestinationVM
> >
> >root@cn-a-0-28:/# vif --get 3
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/3 OS: tapa8f39cc1-01
> > Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
> > Vrf:1 Flags:PL3L2 MTU:9160 Ref:6
> > RX packets:4005754 bytes:3214446570 errors:0
> > TX packets:4912399 bytes:517166392 errors:0
> >
> >
> >root@cn-a-0-28:/# rt --dump 1 | egrep "10.189.85.19/32|Dest"
> >Destination PPL Flags Label Nexthop Stitched
> >MAC(Index)
> >10.189.85.19/32 32 LP 22 35
> >2:59:4f:9c:af:9e(100440)
> >
> >root@cn-a-0-28:/# nh --get 35
> >Id:35 Type:Tunnel Fmly: AF_INET Flags:Valid, MPLSoUDP,
> >Rid:0 Ref_cnt:75 Vrf:0
> > Oif:0 Len:14 Flags Valid, MPLSoUDP, Data:ec f4 bb d9 20
> >e0 ec f4 bb d9 5e c4 08 00
> > Vrf:0 Sip:10.187.1.28 Dip:10.187.1.27
> >
> >root@cn-a-0-28:/# vif --get 0
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/0 OS: bond1 (Speed 10000, Duplex 1)
> > Type:Physical HWaddr:ec:f4:bb:d9:5e:c4 IPaddr:0
> > Vrf:0 Flags:TcL3L2Vp MTU:1514 Ref:8
> > RX packets:5633720 bytes:1088261052 errors:0
> > TX packets:4608455 bytes:3497980076 errors:0
> >
> >
> >#get info about DestinationVM on destination compute node
> >#get vif, rt and nh
> >
> >root@cn-a-0-27:~# vif --get 4
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/4 OS: tap594f9caf-9e
> > Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
> > Vrf:2 Flags:PL3L2D MTU:9160 Ref:6
> > RX packets:167708 bytes:142198914 errors:0
> > TX packets:210585 bytes:16691702 errors:0
> >
> >
> >root@cn-a-0-27:~# rt --dump 2 | egrep "10.189.85.19\/32|Dest"
> >Destination PPL Flags Label Nexthop Stitched
> >MAC(Index)
> >10.189.85.19/32 32 P - 35
> >2:59:4f:9c:af:9e(131480)
> >
> >root@cn-a-0-27:~# nh --get 35
> >Id:35 Type:Encap Fmly: AF_INET Flags:Valid, Policy,
> >Rid:0 Ref_cnt:6 Vrf:2
> > EncapFmly:0806 Oif:4 Len:14 Data:02 59 4f 9c af 9e 00 00
> >5e 00 01 00 08 00
> >
> >
> >#get ProblematicVM vif, rt, nh on destination compute node
> >
> >root@cn-a-0-27:~# vif --list | grep tap0cf19675-22
> >vif0/17 OS: tap0cf19675-22
> >
> >root@cn-a-0-27:~# vif --get 17
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/17 OS: tap0cf19675-22
> > Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
> > Vrf:2 Flags:PL3L2D MTU:9160 Ref:6
> > RX packets:5110 bytes:374284 errors:0
> > TX packets:40265 bytes:2987278 errors:0
> >
> >
> >root@cn-a-0-27:~# rt --dump 2 | egrep "10.189.85.22\/32|Dest"
> >Destination PPL Flags Label Nexthop Stitched
> >MAC(Index)
> >10.189.85.22/32 32 P - 112
> >2:c:f1:96:75:22(197908)
> >
> >root@cn-a-0-27:~# nh --get 112
> >Id:112 Type:Encap Fmly: AF_INET Flags:Valid, Policy,
> >Rid:0 Ref_cnt:6 Vrf:2
> > EncapFmly:0806 Oif:17 Len:14 Data:02 0c f1 96 75 22 00 00
> >5e 00 01 00 08 00
> >
> >
> >
> >
> >Further observations
> >
> >We can't get vif --kernel for tested VM
> >
> >root@cn-a-0-28:/# vif --get 3 --kernel
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >No such file or directory
> >
> >
> >root@cn-a-0-27:~# vif --get 17 --kernel
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >No such file or directory
> >
> >
> >root@cn-a-0-27:~# vif --get 4 --kernel
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >No such file or directory
> >
> >
> >For some we can get this info but is it correct?
> >
> >root@cn-a-0-27:~# vif --get 34
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/34 OS: tap6d1f56cf-c0
> > Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
> > Vrf:1 Flags:PL3L2D MTU:9160 Ref:6
> > RX packets:37583 bytes:23041676 errors:0
> > TX packets:30149 bytes:2358733 errors:0
> >
> >root@cn-a-0-27:~# vif --get 34 --kernel
> >Vrouter Interface Table
> >
> >Flags: P=Policy, X=Cross Connect, S=Service Chain, Mr=Receive Mirror
> > Mt=Transmit Mirror, Tc=Transmit Checksum Offload, L3=Layer 3,
> >L2=Layer 2
> > D=DHCP, Vp=Vhost Physical, Pr=Promiscuous, Vnt=Native Vlan Tagged
> > Mnp=No MAC Proxy, Dpdk=DPDK PMD Interface, Rfl=Receive Filtering
> >Offload, Mon=Interface is Monitored
> > Uuf=Unknown Unicast Flood
> >
> >vif0/4 OS: tap594f9caf-9e
> > Type:Virtual HWaddr:00:00:5e:00:01:00 IPaddr:0
> > Vrf:2 Flags:PL3L2D MTU:9160 Ref:6
> > RX packets:167713 bytes:142199268 errors:0
> > TX packets:210587 bytes:16691786 errors:0
> >
> >
> >Value should be 49 (os_ifindex), am I right?
> >
> >root@cn-a-0-27:~# ip link show | grep tap6d1f56cf-c0
> >49: tap6d1f56cf-c0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
> >pfifo_fast state UP mode DEFAULT group default qlen 500
> >
> >root@cn-a-0-27:~# ip link show | grep tap594f9caf-9e
> >34: tap594f9caf-9e: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc
> >pfifo_fast state UP mode DEFAULT group default qlen 500
> >
> >
> >On http://10.187.0.27:8085/Snh_ItfReq?x=tap594f9caf-9e, os_ifindex looks
> >fine
> >
> >
> >Kind Regards
> >Jacek Czerniak
> >
> >W dniu 14.10.2015 o 07:30, [email protected] pisze:
> >> Send Users mailing list submissions to
> >> [email protected]
> >>
> >> To subscribe or unsubscribe via the World Wide Web, visit
> >>
> >>
> http://lists.opencontrail.org/mailman/listinfo/users_lists.opencontrail.
> >>org
> >>
> >> or, via email, send a message with subject or body 'help' to
> >> [email protected]
> >>
> >> You can reach the person managing the list at
> >> [email protected]
> >>
> >> When replying, please edit your Subject line so it is more specific
> >> than "Re: Contents of Users digest..."
> >>
> >>
> >> Today's Topics:
> >>
> >> 1. Re: Drops of the first SYN packet (Rajagopalan Sivaramakrishnan)
> >>
> >>
> >> ----------------------------------------------------------------------
> >>
> >> Message: 1
> >> Date: Wed, 14 Oct 2015 05:30:39 +0000
> >> From: Rajagopalan Sivaramakrishnan <[email protected]>
> >> To: Piotr P <[email protected]>
> >> Cc: "<[email protected]>" <[email protected]>
> >> Subject: Re: [Users] Drops of the first SYN packet
> >> Message-ID: <d2433529.5b7f3%[email protected]>
> >> Content-Type: text/plain; charset="windows-1252"
> >>
> >> Hi Piotr,
> >> I didn't fully understand your description below, so I?ll describe
> >>the expected behavior. The first packet of the flow should be sent to
> >>the vrouter agent over the pkt0 interface. Subsequently, the agent
> >>should create a flow with the flow action set to forward (which should
> >>be visible in ?flow ?l?). For the packet that gets dropped, do you see
> >>it in ?tcpdump ?ni pkt0 ?x? (you might get some messages from tcpdump
> >>indicating bad header length, but you can ignore those) and ?flow ?l".
> >>Can you please check this on the compute node where you see the SYN
> >>being dropped? Are you seeing issues with ping too or is it only with
> >>TCP SYN?
> >>
> >> Raja
> >>
> >> From: Piotr P <[email protected]<mailto:[email protected]>>
> >> Date: Tuesday, October 13, 2015 at 7:53 AM
> >> To: Rajagopalan Sivaramakrishnan
> >><[email protected]<mailto:[email protected]>>
> >> Cc:
> >>"<[email protected]<mailto:[email protected]>>"
> >><[email protected]<mailto:[email protected]>>
> >> Subject: Re: [Users] Drops of the first SYN packet
> >>
> >>
> >>
> >> Hi Raja,
> >>
> >>
> >> For our tests we took today two new kvm compute nodes with vrouter and
> >>without any Virtual Machines.
> >> What we noticed is that if we spawn up to 4 VM within the same project
> >>and Virtual Network on each compute node everything works all right.
> >> If we add 5th VM on test compute node problem appears , and we can
> >>reproduce this on different physical kvm compute nodes.
> >> On this 5th Virtual machines we are starting seeing first SYN packet
> >>for other VMs located on specific compute node (vrouter) .
> >> All other packets from the transmission is normally delivered to target
> >>virtual machines.
> >> This is visible for all consecutive tcp flows between VM's, and only
> >>for first packet from a new flow.
> >> This 5th VM receives traffic for itself normally. So we end up in
> >>situation where we have affected 4 VM's and one working without problem.
> >> Problem could be resolved if we shutdown virtual port of this 5th VM or
> >>delete it. After this step traffic for other vm is delivered normally.
> >> If we create again new VM problem starts to be visible again.
> >>
> >> We cannot see any new flows between client and 5th VM that receives
> >>additional SYN packets that this VM shouldn't see.
> >> All other flows are created successfully.
> >> In the dropstat output we cannot see any changes especially related to
> >>the drop counter.
> >>
> >>
> >> Following lines you can find example of how we see trafic on this vm
> >>where all first packets are sending
> >>
> >> 192.168.50.22 - client located on first compute node
> >> 192.168.50.27 - 192.168.50.38 - other vm with nginx, located on second
> >>compute node
> >> 192.168.50.33 - VM where all first SYN packets are visible
> >>
> >>
> >> [root@c-cn-28-10 centos]# ip -4 a
> >> 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
> >> inet 127.0.0.1/8<http://127.0.0.1/8> scope host lo
> >> valid_lft forever preferred_lft forever
> >> 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast
> >>state UP qlen 1000
> >> inet 192.168.50.33/20<http://192.168.50.33/20> brd 10.189.63.255
> >>scope global eth0
> >> valid_lft forever preferred_lft forever
> >> [root@c-cn-28-10 centos]# tcpdump "tcp[tcpflags] & (tcp-syn|tcp-ack) !=
> >>0" and port 80
> >> tcpdump: verbose output suppressed, use -v or -vv for full protocol
> >>decode
> >> listening on eth0, link-type EN10MB (Ethernet), capture size 65535 bytes
> >>
> >> 14:20:22.626172 IP 192.168.50.22.51823 > 192.168.50.27.http: Flags [S],
> >>seq 2741907250, win 14600, options [mss 1460,sackOK,TS val 5010347 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:23.631110 IP 192.168.50.22.60984 > 192.168.50.28.http: Flags [S],
> >>seq 3116446858, win 14600, options [mss 1460,sackOK,TS val 5011352 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:24.637322 IP 192.168.50.22.46585 > 192.168.50.29.http: Flags [S],
> >>seq 3566871094, win 14600, options [mss 1460,sackOK,TS val 5012359 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:25.643556 IP 192.168.50.22.41182 > 192.168.50.30.http: Flags [S],
> >>seq 1739097792, win 14600, options [mss 1460,sackOK,TS val 5013365 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:26.651601 IP 192.168.50.22.36921 > 192.168.50.31.http: Flags [S],
> >>seq 3990314127, win 14600, options [mss 1460,sackOK,TS val 5014373 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:27.659212 IP 192.168.50.22.40056 > 192.168.50.32.http: Flags [S],
> >>seq 1091797067, win 14600, options [mss 1460,sackOK,TS val 5015380 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:28.665625 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags [S],
> >>seq 1936544852, win 14600, options [mss 1460,sackOK,TS val 5016387 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:28.665653 IP 192.168.50.33.http > 192.168.50.22.52838: Flags
> >>[S.], seq 2591101890, ack 1936544853, win 14480, options [mss
> >>1460,sackOK,TS val 4294865173 ecr 5016387,nop,wscale 7], length 0
> >> 14:20:28.666021 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags [.],
> >>ack 1, win 115, options [nop,nop,TS val 5016388 ecr 4294865173], length 0
> >> 14:20:28.666046 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags
> >>[P.], seq 1:78, ack 1, win 115, options [nop,nop,TS val 5016388 ecr
> >>4294865173], length 77
> >> 14:20:28.666050 IP 192.168.50.33.http > 192.168.50.22.52838: Flags [.],
> >>ack 78, win 114, options [nop,nop,TS val 4294865173 ecr 5016388], length
> >>0
> >> 14:20:28.666447 IP 192.168.50.33.http > 192.168.50.22.52838: Flags
> >>[P.], seq 1:239, ack 78, win 114, options [nop,nop,TS val 4294865174 ecr
> >>5016388], length 238
> >> 14:20:28.666947 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags [.],
> >>ack 239, win 123, options [nop,nop,TS val 5016389 ecr 4294865174],
> >>length 0
> >> 14:20:28.666958 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags
> >>[F.], seq 78, ack 239, win 123, options [nop,nop,TS val 5016389 ecr
> >>4294865174], length 0
> >> 14:20:28.667002 IP 192.168.50.33.http > 192.168.50.22.52838: Flags
> >>[F.], seq 239, ack 79, win 114, options [nop,nop,TS val 4294865174 ecr
> >>5016389], length 0
> >> 14:20:28.667340 IP 192.168.50.22.52838 > 192.168.50.33.http: Flags [.],
> >>ack 240, win 123, options [nop,nop,TS val 5016390 ecr 4294865174],
> >>length 0
> >> 14:20:28.671229 IP 192.168.50.22.46357 > 192.168.50.34.http: Flags [S],
> >>seq 2912677901, win 14600, options [mss 1460,sackOK,TS val 5016393 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:29.679416 IP 192.168.50.22.53239 > 192.168.50.35.http: Flags [S],
> >>seq 1015724729, win 14600, options [mss 1460,sackOK,TS val 5017401 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:30.687231 IP 192.168.50.22.40427 > 192.168.50.36.http: Flags [S],
> >>seq 3789917763, win 14600, options [mss 1460,sackOK,TS val 5018408 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:31.693362 IP 192.168.50.22.47202 > 192.168.50.37.http: Flags [S],
> >>seq 2700187443, win 14600, options [mss 1460,sackOK,TS val 5019415 ecr
> >>0,nop,wscale 7], length 0
> >> 14:20:32.699384 IP 192.168.50.22.37820 > 192.168.50.38.http: Flags [S],
> >>seq 2458579739, win 14600, options [mss 1460,sackOK,TS val 5020421 ecr
> >>0,nop,wscale 7], length 0
> >>
> >>
> >>
> >> Additioanl test with ping
> >>
> >> [root@client]# ping 192.168.50.35 -c 2
> >> PING 192.168.50.35 (192.168.50.35) 56(84) bytes of data.
> >> 64 bytes from 192.168.50.35<http://192.168.50.35>: icmp_seq=2 ttl=64
> >>time=0.498 ms
> >>
> >> --- 192.168.50.35 ping statistics ---
> >> 2 packets transmitted, 1 received, 50% packet loss, time 999ms
> >> rtt min/avg/max/mdev = 0.498/0.498/0.498/0.000 ms
> >>
> >>
> >> (on "the 5th vm" with interface address 192.168.50.33) tcpdump -i any
> >>icmp -n
> >> 14:39:11.741773 IP 192.168.50.22 > 192.168.50.35<http://192.168.50.35>:
> >>ICMP echo request, id 3626, seq 1, length 64
> >> 14:39:12.685724 IP 192.168.50.22 > 192.168.50.35<http://192.168.50.35>:
> >>ICMP echo request, id 3627, seq 1, length 64
> >> 14:39:13.549434 IP 192.168.50.22 > 192.168.50.35<http://192.168.50.35>:
> >>ICMP echo request, id 3628, seq 1, length 64
> >> 14:39:14.397740 IP 192.168.50.22 > 192.168.50.35<http://192.168.50.35>:
> >>ICMP echo request, id 3629, seq 1, length 64
> >> 14:39:15.237451 IP 192.168.50.22 > 192.168.50.35<http://192.168.50.35>:
> >>ICMP echo request, id 3630, seq 1, length 6
> >>
> >> We are not seeing this icmp packets on the flow list
> >>
> >> [destination compute node "flow -l"
> >> 28268 192.168.50.33:123<http://192.168.50.33:123>
> >>95.158.95.123:123<http://95.158.95.123:123> 17 (1)
> >> 74012 95.158.95.123:123<http://95.158.95.123:123>
> >>192.168.50.33:123<http://192.168.50.33:123> 17 (1)
> >> 238232<=>392568 192.168.50.33:22<http://192.168.50.33:22>
> >>10.163.14.57:47034<http://10.163.14.57:47034> 6 (1)
> >> 392568<=>238232 10.163.14.57:47034<http://10.163.14.57:47034>
> >>192.168.50.33:22<http://192.168.50.33:22> 6 (1)
> >>
> >>
> >>
> >> Please let us know how to troubleshoot it in more details. What we can
> >>check next.
> >>
> >>
> >>
> >> Kind Regards
> >> Piotr Pieprzycki
> >>
> >> On 12 October 2015 at 19:17, Rajagopalan Sivaramakrishnan
> >><[email protected]<mailto:[email protected]>> wrote:
> >> Hi Piotr,
> >> There isn?t a known issue in this respect. Can you please run
> >>?dropstats? on the destination compute node to see if any error counters
> >>are going up inside vrouter?
> >> Also, can you please run ?flow ?l? on the destination compute node to
> >>see if the flow action for the relevant flow is set to F (forward).
> >>
> >> Raja
> >>
> >> From: Users
> >><[email protected]<mailto:
> [email protected]
> >>rail.org>> on behalf of Piotr P
> >><[email protected]<mailto:[email protected]>>
> >> Date: Monday, October 12, 2015 at 7:39 AM
> >> To:
> >>"<[email protected]<mailto:[email protected]>>"
> >><[email protected]<mailto:[email protected]>>
> >> Subject: [Users] Drops of the first SYN packet
> >>
> >>
> >> Hi All,
> >> We are experiencing problem with missing packets between Virtual
> >>Machines located within the same Virtual Network.
> >>
> >> Our setup Contrail 2.20 (buld 64), Priority of encapsulation: VxLAN,
> >>MPLSoUDP, MPLSoGRE
> >>
> >>
> >> The effect is visible during each new transmission/flow. There is
> >>missing first syn packet or first icmp ping.
> >> This affect each tcp session, it's delayed for syn retransmission time.
> >> Drops occurs for each subsequent new tcp session between VM's. We can
> >>repeat tcp connection between VM and first syn will be missing all the
> >>time.
> >> It's not visible in all Virtual Networks. There are some other Virtual
> >>Networks that works correctly.
> >> We can reproduce this behaviour with creating new networks with even
> >>only 2 virtual machines.
> >> Those networks differs only by other routing-target value.
> >>
> >> This effect
> >> does not appear when we have transmission outside VN (eg through
> >>physical gateway)
> >> it's only correlated when traffic stays within network.
> >>
> >> Tcpdump on compute nodes shows that:
> >> - All packets are visible on incoming tap interface (from source VM)
> >> - all packets are visible on bond interfaces on both compute nodes
> >>(vrouter) with VMs (bond active-passive toward physical switch )
> >> Computer nodes are located on the same physical switch (no visible
> >>errors on switch ports)
> >> - first packet/syn from transmission is missing on a tap interface
> >>toward destination VM
> >>
> >>
> >> Please let know if this is some known behaviour. What could be the
> >>steps to troubleshoot this issue?
> >>
> >> Kind Regards
> >> Piotr Pieprzycki
> >>
> >> -------------- next part --------------
> >> An HTML attachment was scrubbed...
> >> URL:
> >><
> http://lists.opencontrail.org/pipermail/users_lists.opencontrail.org/att
> >>achments/20151014/70a5f042/attachment.html>
> >>
> >> ------------------------------
> >>
> >> Subject: Digest Footer
> >>
> >> _______________________________________________
> >> Users mailing list
> >> [email protected]
> >>
> >>
> http://lists.opencontrail.org/mailman/listinfo/users_lists.opencontrail.o
> >>rg
> >>
> >>
> >> ------------------------------
> >>
> >> End of Users Digest, Vol 26, Issue 11
> >> *************************************
> >
> >
> >_______________________________________________
> >Users mailing list
> >[email protected]
> >
> http://lists.opencontrail.org/mailman/listinfo/users_lists.opencontrail.or
> >g
>
>
--
Pozdrawiam,
Jacek Czerniak
_______________________________________________
Users mailing list
[email protected]
http://lists.opencontrail.org/mailman/listinfo/users_lists.opencontrail.org
