Hi, Damjan,
I downloaded the glibc source file. Now, I can see the symbols. See
attachment for all details. Hope they help fix the issue.
Thanks.
Chuan
On Mon, Nov 4, 2019 at 11:00 AM Chuan Han <[email protected]> wrote:
> More details.
>
> warning: core file may not match specified executable file.
> [New LWP 10851]
> [New LWP 10855]
> [New LWP 10853]
> [New LWP 10854]
> [New LWP 10856]
> [New LWP 10857]
> [New LWP 10858]
> [New LWP 10852]
> [New LWP 10859]
> [New LWP 10860]
> [New LWP 10861]
> [Thread debugging using libthread_db enabled]
> Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
> Core was generated by `vpp -c vpp_startup/startup.conf'.
> Program terminated with signal SIGABRT, Aborted.
> #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
> 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
> [Current thread is 1 (Thread 0x7f8120c00780 (LWP 10851))]
> (gdb)
>
> On Mon, Nov 4, 2019 at 10:46 AM Chuan Han <[email protected]> wrote:
>
>> Not sure how to get all symbols. Can someone share some steps?
>>
>> (gdb) list
>>
>>
>>
>> 46 in ../sysdeps/unix/sysv/linux/raise.c
>>
>>
>>
>> (gdb) bt full
>>
>>
>>
>> #0 __GI_raise (sig=sig@entry=6) at
>> ../sysdeps/unix/sysv/linux/raise.c:51
>>
>>
>>
>> set = {__val = {1024, 140192549297216, 140191458925760,
>> 94479452800528, 140191458925744, 14738652586040953856, 12, 94479452800528,
>> 0, 14738652586040953856, 12, 1, 94479452800528, 1, 12, 140192554943764}}
>> pid = <optimized out>
>> tid = <optimized out>
>> ret = <optimized out>
>> #1 0x00007f811edf2801 in __GI_abort () at abort.c:79
>> save_stage = 1
>> act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction =
>> 0x0}, sa_mask = {__val = {140192549484596, 140192549745376,
>> 140192549484596, 140192554638615, 140192554894358, 140192549484596,
>> 140191451682880, 4294967295, 4, 1024, 140191451683548, 140191451691708,
>> 14738652586040953856,
>> 140191436101232, 8, 8}}, sa_flags = 853347328, sa_restorer
>> = 0x7f80de1c1ef0}
>> sigs = {__val = {32, 0 <repeats 15 times>}}
>> __cnt = <optimized out>
>> __set = <optimized out>
>> __cnt = <optimized out>
>> __set = <optimized out>
>> #2 0x000055edb4f0ad44 in os_exit ()
>> No symbol table info available.
>> #3 0x00007f811f6f53d9 in ?? () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #4 <signal handler called>
>> No locals.
>> #5 0x00007f811f6db793 in ?? () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #6 0x00007f811f6df4d9 in ?? () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #7 0x00007f811f6a4eee in vlib_call_init_exit_functions () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #8 0x00007f811f6b5d17 in vlib_main () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #9 0x00007f811f6f4416 in ?? () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>> No symbol table info available.
>> #10 0x00007f811f1cb834 in clib_calljmp () from
>> /usr/lib/x86_64-linux-gnu/libvppinfra.so.19.08.1
>> No symbol table info available.
>> #11 0x00007ffe3cb01770 in ?? ()
>> No symbol table info available.
>> #12 0x00007f811f6f586f in vlib_unix_main () from
>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>
>> On Mon, Nov 4, 2019 at 10:45 AM Chuan Han via Lists.Fd.Io <chuanhan=
>> [email protected]> wrote:
>>
>>> Here is the gdb vpp core output
>>>
>>> (gdb) f
>>>
>>>
>>>
>>> #0 __GI_raise (sig=sig@entry=6) at
>>> ../sysdeps/unix/sysv/linux/raise.c:51
>>>
>>>
>>>
>>> 51 in ../sysdeps/unix/sysv/linux/raise.c
>>>
>>>
>>>
>>> (gdb) bt
>>>
>>>
>>>
>>> #0 __GI_raise (sig=sig@entry=6) at
>>> ../sysdeps/unix/sysv/linux/raise.c:51
>>>
>>>
>>>
>>> #1 0x00007f811edf2801 in __GI_abort () at abort.c:79
>>> #2 0x000055edb4f0ad44 in os_exit ()
>>> #3 0x00007f811f6f53d9 in ?? () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #4 <signal handler called>
>>> #5 0x00007f811f6db793 in ?? () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #6 0x00007f811f6df4d9 in ?? () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #7 0x00007f811f6a4eee in vlib_call_init_exit_functions () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #8 0x00007f811f6b5d17 in vlib_main () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #9 0x00007f811f6f4416 in ?? () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #10 0x00007f811f1cb834 in clib_calljmp () from
>>> /usr/lib/x86_64-linux-gnu/libvppinfra.so.19.08.1
>>> #11 0x00007ffe3cb01770 in ?? ()
>>> #12 0x00007f811f6f586f in vlib_unix_main () from
>>> /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
>>> #13 0x3de8f63100000400 in ?? ()
>>> #14 0x8100458b49ffd49d in ?? ()
>>> #15 0x8b4800000100f868 in ?? ()
>>> #16 0x6d8141fffffb6085 in ?? ()
>>> #17 0x408b480000010008 in ?? ()
>>> #18 0x480000000000c740 in ?? ()
>>>
>>> let me know more specific steps to pinpoint the issues.
>>>
>>> On Mon, Nov 4, 2019 at 10:13 AM Damjan Marion <[email protected]> wrote:
>>>
>>>> i remember doing corelist-workers with > 50 cores…..
>>>>
>>>> If you paste traceback we may have better clue what is wrong…
>>>>
>>>> —
>>>> Damjan
>>>>
>>>>
>>>>
>>>> On 4 Nov 2019, at 18:51, Chuan Han via Lists.Fd.Io <
>>>> [email protected]> wrote:
>>>>
>>>> All even number cores are on numa 0, which also hosts all nics.
>>>>
>>>> It seems corelist-workers can only take maximum 8 cores.
>>>>
>>>> On Mon, Nov 4, 2019 at 9:45 AM Tkachuk, Georgii <
>>>> [email protected]> wrote:
>>>>
>>>>> Hi Chuan, are cores 20 and 22 on socket0 or socket1? If they are on
>>>>> socket1, the application is crashing because the aesni_mb driver is
>>>>> pointing to socket0: vdev crypto_aesni_mb0,socket_id=0.
>>>>>
>>>>>
>>>>>
>>>>> George
>>>>>
>>>>>
>>>>>
>>>>> *From:* [email protected] <[email protected]> *On Behalf Of *Chuan
>>>>> Han via Lists.Fd.Io <http://lists.fd.io/>
>>>>> *Sent:* Monday, November 04, 2019 10:27 AM
>>>>> *To:* vpp-dev <[email protected]>
>>>>> *Cc:* [email protected]
>>>>> *Subject:* [vpp-dev] Is there a limit when assigning corelist-workers
>>>>> in vpp?
>>>>>
>>>>>
>>>>>
>>>>> Hi, vpp experts,
>>>>>
>>>>>
>>>>>
>>>>> I am trying to allocate more cores to a phy nic. I want to allocate
>>>>> cores 4,6,8,10 to eth0, and cores 12,14,16,18 to eth1.
>>>>>
>>>>>
>>>>>
>>>>> cpu {
>>>>> main-core 2
>>>>> * # corelist-workers 4,6,8,10,12,14,16,18,20,22 <== This does not
>>>>> work. vpp crashes when starting. *
>>>>> corelist-workers 4,6,8,10,12,14,16,18
>>>>> }
>>>>>
>>>>> dpdk {
>>>>> socket-mem 2048,0
>>>>> log-level debug
>>>>> no-tx-checksum-offload
>>>>> dev default{
>>>>> num-tx-desc 512
>>>>> num-rx-desc 512
>>>>> }
>>>>> dev 0000:1a:00.0 {
>>>>> # workers 4,6,8,10,12
>>>>> workers 4,6,8,10
>>>>> name eth0
>>>>> }
>>>>> dev 0000:19:00.1 {
>>>>> # workers 14,16,18,20,22
>>>>> workers 12,14,16,18
>>>>> name eth1
>>>>> }
>>>>> # Use aesni mb lib.
>>>>> vdev crypto_aesni_mb0,socket_id=0
>>>>> # Use qat VF pcie addresses.
>>>>> # dev 0000:3d:01.0
>>>>> no-multi-seg
>>>>> }
>>>>>
>>>>>
>>>>>
>>>>> Afte vpp starts, I can see eth1 got 4 cores but eth0 only got 3 cores.
>>>>>
>>>>>
>>>>>
>>>>> vpp# sh thread
>>>>> ID Name Type LWP Sched Policy (Priority)
>>>>> lcore Core Socket State
>>>>> 0 vpp_main 10653 other (0)
>>>>> 2 0 0
>>>>> 1 vpp_wk_0 workers 10655 other (0)
>>>>> 4 1 0
>>>>> 2 vpp_wk_1 workers 10656 other (0)
>>>>> 6 4 0
>>>>> 3 vpp_wk_2 workers 10657 other (0)
>>>>> 8 2 0
>>>>> 4 vpp_wk_3 workers 10658 other (0)
>>>>> 10 3 0
>>>>> 5 vpp_wk_4 workers 10659 other (0)
>>>>> 12 8 0
>>>>> 6 vpp_wk_5 workers 10660 other (0)
>>>>> 14 13 0
>>>>> 7 vpp_wk_6 workers 10661 other (0)
>>>>> 16 9 0
>>>>> *8 vpp_wk_7 workers 10662 other (0)
>>>>> 18 12 0 <=== core 18 is not used by eth0.*
>>>>> vpp# sh interface rx-placement
>>>>> Thread 1 (vpp_wk_0):
>>>>> node dpdk-input:
>>>>> eth1 queue 0 (polling)
>>>>> Thread 2 (vpp_wk_1):
>>>>> node dpdk-input:
>>>>> eth1 queue 1 (polling)
>>>>> Thread 3 (vpp_wk_2):
>>>>> node dpdk-input:
>>>>> eth1 queue 2 (polling)
>>>>> Thread 4 (vpp_wk_3):
>>>>> node dpdk-input:
>>>>> eth1 queue 3 (polling)
>>>>>
>>>>>
>>>>>
>>>>> *Thread 5 (vpp_wk_4): node dpdk-input: eth0 queue 0 (polling)
>>>>> eth0 queue 2 (polling)*
>>>>> Thread 6 (vpp_wk_5):
>>>>> node dpdk-input:
>>>>> eth0 queue 3 (polling)
>>>>> Thread 7 (vpp_wk_6):
>>>>> node dpdk-input:
>>>>> eth0 queue 1 (polling)
>>>>> vpp#
>>>>>
>>>>>
>>>>>
>>>>> It seems there is a limitation on assigning cores to nic.
>>>>>
>>>>> 1. I cannot allocate cores after 20 to corelist-workers.
>>>>>
>>>>> 2. Cores after 18 cannot be allocated to nic.
>>>>>
>>>>>
>>>>>
>>>>> Is this some bug? Or, some undocumented limitation?
>>>>>
>>>>>
>>>>>
>>>>> Thanks.
>>>>>
>>>>> Chuan
>>>>>
>>>> -=-=-=-=-=-=-=-=-=-=-=-
>>>> Links: You receive all messages sent to this group.
>>>>
>>>> View/Reply Online (#14491): https://lists.fd.io/g/vpp-dev/message/14491
>>>> Mute This Topic: https://lists.fd.io/mt/41334952/675642
>>>> Group Owner: [email protected]
>>>> Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [[email protected]]
>>>> -=-=-=-=-=-=-=-=-=-=-=-
>>>>
>>>>
>>>> -=-=-=-=-=-=-=-=-=-=-=-
>>> Links: You receive all messages sent to this group.
>>>
>>> View/Reply Online (#14494): https://lists.fd.io/g/vpp-dev/message/14494
>>> Mute This Topic: https://lists.fd.io/mt/41334952/1991531
>>> Group Owner: [email protected]
>>> Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [[email protected]]
>>> -=-=-=-=-=-=-=-=-=-=-=-
>>>
>>
(gdb) set substitute-path /build/glibc-OTsEL5/glibc-2.27 /opt/src/glibc-2.27
(gdb) list
46 int ret = INLINE_SYSCALL (tgkill, 3, pid, tid, sig);
47
48 __libc_signal_restore_set (&set);
49
50 return ret;
51 }
52 libc_hidden_def (raise)
53 weak_alias (raise, gsignal)
(gdb) bt
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007f811edf2801 in __GI_abort () at abort.c:79
#2 0x000055edb4f0ad44 in os_exit ()
#3 0x00007f811f6f53d9 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#4 <signal handler called>
#5 0x00007f811f6db793 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#6 0x00007f811f6df4d9 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#7 0x00007f811f6a4eee in vlib_call_init_exit_functions () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#8 0x00007f811f6b5d17 in vlib_main () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#9 0x00007f811f6f4416 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#10 0x00007f811f1cb834 in clib_calljmp () from /usr/lib/x86_64-linux-gnu/libvppinfra.so.19.08.1
#11 0x00007ffe3cb01770 in ?? ()
#12 0x00007f811f6f586f in vlib_unix_main () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
#13 0x3de8f63100000400 in ?? ()
(gdb) f 3
#3 0x00007f811f6f53d9 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
74 int save_stage = stage;
75
76 stage = 0;
77 __libc_lock_unlock_recursive (lock);
78
79 raise (SIGABRT);
80
81 __libc_lock_lock_recursive (lock);
82 stage = save_stage + 1;
83 }
(gdb) f 5
#5 0x00007f811f6db793 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
84
85 /* There was a handler installed. Now remove it. */
86 if (stage == 2)
87 {
88 ++stage;
89 memset (&act, '\0', sizeof (struct sigaction));
90 act.sa_handler = SIG_DFL;
91 __sigfillset (&act.sa_mask);
92 act.sa_flags = 0;
93 __sigaction (SIGABRT, &act, NULL);
(gdb) f 6
#6 0x00007f811f6df4d9 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
94 }
95
96 /* Try again. */
97 if (stage == 3)
98 {
99 ++stage;
100 raise (SIGABRT);
101 }
102
103 /* Now try to abort using the system specific command. */
(gdb) f 7
#7 0x00007f811f6a4eee in vlib_call_init_exit_functions () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
104 if (stage == 4)
105 {
106 ++stage;
107 ABORT_INSTRUCTION;
108 }
109
110 /* If we can't signal ourselves and the abort instruction failed, exit. */
111 if (stage == 5)
112 {
113 ++stage;
(gdb) f 8
#8 0x00007f811f6b5d17 in vlib_main () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
114 _exit (127);
115 }
116
117 /* If even this fails try to use the provided instruction to crash
118 or otherwise make sure we never return. */
119 while (1)
120 /* Try for ever and ever. */
121 ABORT_INSTRUCTION;
122 }
123 libc_hidden_def (abort)
(gdb) f 10
#10 0x00007f811f1cb834 in clib_calljmp () from /usr/lib/x86_64-linux-gnu/libvppinfra.so.19.08.1
(gdb) list
74 int save_stage = stage;
75
76 stage = 0;
77 __libc_lock_unlock_recursive (lock);
78
79 raise (SIGABRT);
80
81 __libc_lock_lock_recursive (lock);
82 stage = save_stage + 1;
83 }
(gdb) f 9
#9 0x00007f811f6f4416 in ?? () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
84
85 /* There was a handler installed. Now remove it. */
86 if (stage == 2)
87 {
88 ++stage;
89 memset (&act, '\0', sizeof (struct sigaction));
90 act.sa_handler = SIG_DFL;
91 __sigfillset (&act.sa_mask);
92 act.sa_flags = 0;
93 __sigaction (SIGABRT, &act, NULL);
(gdb) f 12
#12 0x00007f811f6f586f in vlib_unix_main () from /usr/lib/x86_64-linux-gnu/libvlib.so.19.08.1
(gdb) list
94 }
95
96 /* Try again. */
97 if (stage == 3)
98 {
99 ++stage;
100 raise (SIGABRT);
101 }
102
103
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#14497): https://lists.fd.io/g/vpp-dev/message/14497
Mute This Topic: https://lists.fd.io/mt/41334952/21656
Group Owner: [email protected]
Unsubscribe: https://lists.fd.io/g/vpp-dev/unsub [[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
