Hello!
We have made the switch fairly recently to using UHD v4 with our X310 who has 2
UBX160s. We have it using the XG firmware image, with both SFP+ ports on the
radio connected to an SFP+ card on our workstation.
Initially we tried out v4.1.0.5 and noticed that once in a while our
application would segfault and crash. Maybe once every day or so. We updated to
v4.3.0.0 and then v4.4.0.0 and were seeing the same issue. I created a short
little test program that seems to replicate that intermittent segfault issue.
The segfault does not seem related to the center frequency of the collects, or
if we are also transmitting at the same time or not. The issue only seems to
happen when trying to do receives from each daughtercard at the same time in
separate threads (one daughtercard per thread), but maybe I just have not seen
it yet when only doing single-threaded receives from a single daughtercard. We
seem to hit this issue maybe once in every couple thousand or so "make
rx_streamer then receive" iterations. It seems to segfault in the same place in
the rx_streamer creation process each time.
I was also able to replicate the segfault with that test program on a different
workstation, so I think it is unlikely that the issue is caused by a bad kit of
RAM or some such in the workstation.
Attached is the test program source, command used to compile the test program,
and output (stdout+stderr) from that test program when we encounter the
segfault. I've also attached the steps we take to compile UHD if that helps.
That segmentation fault seems to happen when creating an rx_streamer and seems
unrelated to what I am doing in the program, but maybe that's not true. Is
there maybe something wrong with what I am doing here in this test program?
Thank you,
Andrew Fountain
Black River Systems Co., Inc.
162 Genesee Street
Utica, NY 13502
#include <uhd/usrp/multi_usrp.hpp>
#include <string>
#include <cstdio>
#include <thread>
#include <vector>
#define REPEAT(statement, exc) \
do { \
while (true) { \
try { \
statement; \
break; \
} catch (exc e) { \
printf("Caught exception: \n\t%s\n"\
"Retrying...\n", e.what()); \
std::this_thread::sleep_for( \
std::chrono::milliseconds(1) \
); \
} \
} \
} while (0)
uhd::usrp::multi_usrp::sptr make_usrp()
{
uhd::usrp::multi_usrp::sptr usrp =
uhd::usrp::multi_usrp::make(std::string("addr=192.168.30.2,second_addr=192.168.40.2"));
std::string subdev("A:0 B:0");
usrp->set_rx_subdev_spec(subdev);
return usrp;
}
void setup_rx(uhd::usrp::multi_usrp::sptr usrp, size_t channel, double
sampling, double freq, double gain,
std::string antenna)
{
usrp->set_rx_rate(sampling, channel);
usrp->set_rx_freq(uhd::tune_request_t(freq, -sampling / 2), channel);
//offset tuning
usrp->set_rx_gain(gain, channel);
usrp->set_rx_antenna(antenna.c_str(), channel);
std::this_thread::sleep_for(std::chrono::seconds(1));
uhd::sensor_value_t lo_locked = usrp->get_rx_sensor("lo_locked", channel);
UHD_ASSERT_THROW(lo_locked.to_bool());
}
void rx_thd_target(uhd::usrp::multi_usrp::sptr usrp, std::string antenna,
size_t channel, size_t samples_per_buffer, double sampling, double freq,
double gain,
double duration, std::vector<double> rx_times)
{
//tune radio
setup_rx(usrp, channel, sampling, freq, gain, antenna);
//create receive buffer
std::vector<std::complex<short>> rx_buf(samples_per_buffer);
size_t rx_idx = 0;
for (const double &rx_time : rx_times)
{
++rx_idx;
//skip any receives we're late for
if (usrp->get_time_now().get_real_secs() > rx_time)
{
printf("Skipped late rx %04ld\n", rx_idx);
continue;
}
bool rx_done = false;
std::vector<size_t> buf_sizes;
//interleaved complex int16 RX
uhd::stream_args_t stream_args = uhd::stream_args_t("sc16", "sc16");
stream_args.channels = {channel};
//try to make RX streamer.
// sometimes we get a timeout error, so retry if needed
uhd::rx_streamer::sptr streamer;
REPEAT(streamer = usrp->get_rx_stream(stream_args), uhd::io_error);
//tell USRP to start receiving at the specified time
uhd::time_spec_t start_time = uhd::time_spec_t(rx_time);
uhd::time_spec_t stop_time = start_time + duration;
uhd::stream_cmd_t
stream_cmd(uhd::stream_cmd_t::STREAM_MODE_START_CONTINUOUS);
stream_cmd.stream_now = false;
stream_cmd.time_spec = start_time;
streamer->issue_stream_cmd(stream_cmd);
uhd::rx_metadata_t md;
while (!rx_done)
{
//get samples from USRP
size_t num_rx_samps = streamer->recv(&rx_buf.front(), samples_per_buffer,
md, 0.5);
buf_sizes.push_back(num_rx_samps);
//check for done
auto now = usrp->get_time_now();
if (now >= stop_time)
{
rx_done = true;
}
}
//stream cleanup
uhd::stream_cmd_t
stream_cmd_end(uhd::stream_cmd_t::STREAM_MODE_STOP_CONTINUOUS);
streamer->issue_stream_cmd(stream_cmd_end);
streamer.reset();
if ((rx_idx % 10) == 0)
{
printf("rx %04ld complete!\n", rx_idx);
}
}
}
int main()
{
auto usrp = make_usrp();
double sampling = 100e6;
double gain = 0;
double rx_duration = 0.25; //s
size_t num_rx = 1000;
double collect_gap = 0.125; //s
//do a number of receives, with gaps in-between each collect
std::vector<double> rx_times;
double t0 = usrp->get_time_now().get_real_secs() + 2.0; //2s start delay
for (size_t i = 0; i < num_rx; ++i)
{
rx_times.push_back(t0 + i*(rx_duration + collect_gap));
}
//start threads who do repeated file receive from each daughtercard
std::vector<size_t> rx_channels{0,1};
size_t num_channels = rx_channels.size();
std::vector<std::string> rx_antennas{"RX2", "RX2"};
std::vector<double> rx_freqs{456e6, 789e6};
size_t rx_spb = 8388608;
std::vector<std::thread> rx_thds;
for (size_t i = 0; i < num_channels; ++i)
{
rx_thds.emplace_back(&rx_thd_target, usrp, rx_antennas[i], rx_channels[i],
rx_spb, sampling, rx_freqs[i], gain, rx_duration, rx_times);
}
for (std::thread &thd : rx_thds)
{
thd.join();
}
printf("Utility complete. Did %ld total receives from %ld channel(s)!\n",
num_rx * num_channels, num_channels);
return 0;
}g++ rx_streamer_segfault.cpp -g -std=c++17 -fpic -fsanitize=address
-static-libasan \
-I /usr/local/uhd4/include/ -luhd -L /usr/local/uhd4/lib64 \
-Wl,-rpath=/usr/local/uhd4/lib64 -o rx_streamer_segfault.exe
$ ./rx_streamer_segfault.exe
[LOG] Failed to set log level to: 6[INFO] [UHD] linux; GNU C++ version 11.2.1
20220127 (Red Hat 11.2.1-9); Boost_108100; UHD_4.4.0.0-0-unknown
[INFO] [X300] X300 initialization sequence...
[INFO] [X300] Maximum frame size: 8000 bytes.
[INFO] [X300] Maximum frame size: 8000 bytes.
[INFO] [GPS] Found an internal GPSDO: LC_XO, Firmware Rev 0.929b
[INFO] [X300] Radio 1x clock: 200 MHz
[WARNING] [RFNOC::GRAPH] One or more blocks timed out during flush!
[WARNING] [UHD] The environment variable UHD_CONFIG_DIR is deprecated. Refer to
https://files.ettus.com/manual/page_calibration.html for how to store
calibration data.
[WARNING] [UHD] The environment variable UHD_CONFIG_DIR is deprecated. Refer to
https://files.ettus.com/manual/page_calibration.html for how to store
calibration data.
[WARNING] [0/Radio#0] Attempting to set tick rate to 0. Skipping.
rx 0010 complete!
rx 0010 complete!
rx 0020 complete!
rx 0020 complete!
rx 0030 complete!
rx 0030 complete!
rx 0040 complete!
rx 0040 complete!
rx 0050 complete!
rx 0050 complete!
rx 0060 complete!
rx 0060 complete!
rx 0070 complete!
rx 0070 complete!
rx 0080 complete!
rx 0080 complete!
rx 0090 complete!
rx 0090 complete!
rx 0100 complete!
rx 0100 complete!
rx 0110 complete!
rx 0110 complete!
rx 0120 complete!
rx 0120 complete!
rx 0130 complete!
rx 0130 complete!
rx 0140 complete!
rx 0140 complete!
rx 0150 complete!
rx 0150 complete!
rx 0160 complete!
rx 0160 complete!
rx 0170 complete!
rx 0170 complete!
rx 0180 complete!
rx 0180 complete!
rx 0190 complete!
rx 0190 complete!
rx 0200 complete!
rx 0200 complete!
rx 0210 complete!
rx 0210 complete!
rx 0220 complete!
rx 0220 complete!
rx 0230 complete!
rx 0230 complete!
rx 0240 complete!
rx 0240 complete!
rx 0250 complete!
rx 0250 complete!
rx 0260 complete!
rx 0260 complete!
rx 0270 complete!
rx 0270 complete!
rx 0280 complete!
rx 0280 complete!
rx 0290 complete!
rx 0290 complete!
rx 0300 complete!
rx 0300 complete!
rx 0310 complete!
rx 0310 complete!
rx 0320 complete!
rx 0320 complete!
rx 0330 complete!
rx 0330 complete!
rx 0340 complete!
rx 0340 complete!
rx 0350 complete!
rx 0350 complete!
rx 0360 complete!
rx 0360 complete!
rx 0370 complete!
rx 0370 complete!
rx 0380 complete!
rx 0380 complete!
rx 0390 complete!
rx 0390 complete!
rx 0400 complete!
rx 0400 complete!
rx 0410 complete!
rx 0410 complete!
rx 0420 complete!
rx 0420 complete!
rx 0430 complete!
rx 0430 complete!
rx 0440 complete!
rx 0440 complete!
rx 0450 complete!
rx 0450 complete!
rx 0460 complete!
rx 0460 complete!
rx 0470 complete!
rx 0470 complete!
rx 0480 complete!
rx 0480 complete!
rx 0490 complete!
rx 0490 complete!
rx 0500 complete!
rx 0500 complete!
rx 0510 complete!
rx 0510 complete!
rx 0520 complete!
rx 0520 complete!
rx 0530 complete!
rx 0530 complete!
rx 0540 complete!
rx 0540 complete!
rx 0550 complete!
rx 0550 complete!
rx 0560 complete!
rx 0560 complete!
rx 0570 complete!
rx 0570 complete!
rx 0580 complete!
rx 0580 complete!
rx 0590 complete!
rx 0590 complete!
rx 0600 complete!
rx 0600 complete!
rx 0610 complete!
rx 0610 complete!
rx 0620 complete!
rx 0620 complete!
rx 0630 complete!
rx 0630 complete!
rx 0640 complete!
rx 0640 complete!
rx 0650 complete!
rx 0650 complete!
rx 0660 complete!
rx 0660 complete!
AddressSanitizer:DEADLYSIGNAL
=================================================================
==1268==ERROR: AddressSanitizer: SEGV on unknown address 0xffffffffffffffe7 (pc
0x7f46346a0c77 bp 0x615000021bc8 sp 0x7f462dae6ed0 T5)
==1268==The signal is caused by a READ memory access.
#0 0x7f46346a0c77 in std::string::compare(std::string const&) const [clone
.isra.0] (/usr/uhd4/lib64/libuhd.so.4.4.0+0x282c77)
#1 0x7f46346a284e in std::_Rb_tree<std::string, std::pair<std::string
const, (anonymous namespace)::streamer_info_t>,
std::_Select1st<std::pair<std::string const, (anonymous
namespace)::streamer_info_t> >, std::less<std::string>,
std::allocator<std::pair<std::string const, (anonymous
namespace)::streamer_info_t> >
>::_M_get_insert_hint_unique_pos(std::_Rb_tree_const_iterator<std::pair<std::string
const, (anonymous namespace)::streamer_info_t> >, std::string const&)
(/usr/uhd4/lib64/libuhd.so.4.4.0+0x28484e)
#2 0x7f46346a2cfa in std::map<std::string, (anonymous
namespace)::streamer_info_t, std::less<std::string>,
std::allocator<std::pair<std::string const, (anonymous
namespace)::streamer_info_t> > >::operator[](std::string&&)
(/usr/uhd4/lib64/libuhd.so.4.4.0+0x284cfa)
#3 0x7f46346ae2ce in rfnoc_graph_impl::connect(uhd::rfnoc::block_id_t
const&, unsigned long, std::shared_ptr<uhd::rx_streamer>, unsigned long,
unsigned long) (/usr/uhd4/lib64/libuhd.so.4.4.0+0x2902ce)
#4 0x7f463479da1b in multi_usrp_rfnoc::get_rx_stream(uhd::stream_args_t
const&) (/usr/uhd4/lib64/libuhd.so.4.4.0+0x37fa1b)
#5 0x4e99d6 in rx_thd_target(std::shared_ptr<uhd::usrp::multi_usrp>,
std::string, unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> >)
cpp/test/rx_streamer_segfault.cpp:80
#6 0x4fc118 in void std::__invoke_impl<void, void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > >(std::__invoke_other, void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&&,
std::string&&, unsigned long&&, unsigned long&&, double&&, double&&, double&&,
double&&, std::vector<double, std::allocator<double> >&&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/invoke.h:61
#7 0x4fbc98 in std::__invoke_result<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > >::type std::__invoke<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > >(void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&&,
std::string&&, unsigned long&&, unsigned long&&, double&&, double&&, double&&,
double&&, std::vector<double, std::allocator<double> >&&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/invoke.h:96
#8 0x4fba43 in void std::thread::_Invoker<std::tuple<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > > >::_M_invoke<0ul, 1ul, 2ul, 3ul,
4ul, 5ul, 6ul, 7ul, 8ul, 9ul>(std::_Index_tuple<0ul, 1ul, 2ul, 3ul, 4ul, 5ul,
6ul, 7ul, 8ul, 9ul>)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/std_thread.h:253
#9 0x4fb921 in std::thread::_Invoker<std::tuple<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > > >::operator()()
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/std_thread.h:260
#10 0x4fb905 in
std::thread::_State_impl<std::thread::_Invoker<std::tuple<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>, std::string,
unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> > > > >::_M_run()
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/std_thread.h:211
#11 0x7f4634aaf3a3 in execute_native_thread_routine
(/usr/uhd4/lib64/libuhd.so.4.4.0+0x6913a3)
#12 0x7f46337f3ea4 (/lib64/libpthread.so.0+0x7ea4)
#13 0x7f4633306b0c in clone (/lib64/libc.so.6+0xfeb0c)
AddressSanitizer can not provide additional info.
SUMMARY: AddressSanitizer: SEGV (/usr/uhd4/lib64/libuhd.so.4.4.0+0x282c77) in
std::string::compare(std::string const&) const [clone .isra.0]
Thread T5 created by T0 here:
#0 0x4454d5 in pthread_create
(/testbin/build/uhd_test/rx_streamer_segfault.exe+0x4454d5)
#1 0x7f4634aaf649 in
std::thread::_M_start_thread(std::unique_ptr<std::thread::_State,
std::default_delete<std::thread::_State> >, void (*)())
(/usr/uhd4/lib64/libuhd.so.4.4.0+0x691649)
#2 0x4f649a in void
__gnu_cxx::new_allocator<std::thread>::construct<std::thread, void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&>(std::thread*, void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/ext/new_allocator.h:162
#3 0x4f3dad in void std::allocator_traits<std::allocator<std::thread>
>::construct<std::thread, void (*)(std::shared_ptr<uhd::usrp::multi_usrp>,
std::string, unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> >),
std::shared_ptr<uhd::usrp::multi_usrp>&, std::string&, unsigned long&, unsigned
long&, double&, double&, double&, double&, std::vector<double,
std::allocator<double> >&>(std::allocator<std::thread>&, std::thread*, void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/alloc_traits.h:516
#4 0x4f40bb in void std::vector<std::thread, std::allocator<std::thread>
>::_M_realloc_insert<void (*)(std::shared_ptr<uhd::usrp::multi_usrp>,
std::string, unsigned long, unsigned long, double, double, double, double,
std::vector<double, std::allocator<double> >),
std::shared_ptr<uhd::usrp::multi_usrp>&, std::string&, unsigned long&, unsigned
long&, double&, double&, double&, double&, std::vector<double,
std::allocator<double> >&>(__gnu_cxx::__normal_iterator<std::thread*,
std::vector<std::thread, std::allocator<std::thread> > >, void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/vector.tcc:449
#5 0x4f09c1 in std::thread& std::vector<std::thread,
std::allocator<std::thread> >::emplace_back<void
(*)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&>(void
(*&&)(std::shared_ptr<uhd::usrp::multi_usrp>, std::string, unsigned long,
unsigned long, double, double, double, double, std::vector<double,
std::allocator<double> >), std::shared_ptr<uhd::usrp::multi_usrp>&,
std::string&, unsigned long&, unsigned long&, double&, double&, double&,
double&, std::vector<double, std::allocator<double> >&)
/opt/rh/devtoolset-11/root/usr/include/c++/11/bits/vector.tcc:121
#6 0x4eb1b0 in main cpp/test/rx_streamer_segfault.cpp:149
#7 0x7f463322a554 in __libc_start_main (/lib64/libc.so.6+0x22554)
==1268==ABORTING
mkdir /uhd_install && cd /uhd_install &&\
wget https://github.com/EttusResearch/uhd/archive/refs/tags/v4.4.0.0.tar.gz
&&\
tar -xf *.tar.gz && cd uhd-*/host &&\
mkdir build && cd build &&\
. /opt/rh/devtoolset-11/enable &&\
cmake ../ \
-DENABLE_X300=ON \
-DENABLE_UTILS=ON \
-DENABLE_EXAMPLES=ON \
-DENABLE_E300=OFF \
-DENABLE_E320=OFF \
-DENABLE_B200=OFF \
-DENABLE_B100=OFF \
-DENABLE_N230=OFF \
-DENABLE_N300=OFF \
-DENABLE_N320=OFF \
-DENABLE_X400=OFF \
-DENABLE_MAN_PAGES=OFF \
-DENABLE_DOXYGEN=OFF \
-DENABLE_MANUAL=OFF \
-DENABLE_TESTS=OFF \
-DENABLE_USB=OFF \
-DENABLE_OCTOCLOCK=OFF \
-DENABLE_MPMD=OFF\
-DENABLE_SIM=OFF\
-Wno-dev \
-DCMAKE_C_FLAGS="-Ofast -mtune=intel" \
-DCMAKE_CXX_FLAGS="-Ofast -mtune=intel" \
-DBoost_USE_STATIC_LIBS=OFF \
-DBoost_USE_STATIC_RUNTIME=OFF \
-DBoost_NO_BOOST_CMAKE=ON \
-DENABLE_STATIC_LIBS=OFF \
-DENABLE_PYTHON_API=ON \
-DCMAKE_BUILD_TYPE=Release \
-DLIB_SUFFIX=64 \
-DCMAKE_INSTALL_PREFIX=/usr/local/uhd4 &&\
make -j16 && make install && rm -rf /uhd_install_______________________________________________
USRP-users mailing list -- usrp-users@lists.ettus.com
To unsubscribe send an email to usrp-users-le...@lists.ettus.com
